diff options
37 files changed, 610 insertions, 172 deletions
diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 0ee80cf114..c0b7a969c0 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -45,6 +45,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/init"\ " src/kernel"\ " src/node/chainstate.cpp"\ + " src/node/chainstatemanager_args.cpp"\ " src/node/mempool_args.cpp"\ " src/node/validation_cache_args.cpp"\ " src/policy/feerate.cpp"\ diff --git a/src/Makefile.am b/src/Makefile.am index 70a0ca8915..6d10f86d57 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -198,6 +198,7 @@ BITCOIN_CORE_H = \ node/blockstorage.h \ node/caches.h \ node/chainstate.h \ + node/chainstatemanager_args.h \ node/coin.h \ node/connection_types.h \ node/context.h \ @@ -381,6 +382,7 @@ libbitcoin_node_a_SOURCES = \ node/blockstorage.cpp \ node/caches.cpp \ node/chainstate.cpp \ + node/chainstatemanager_args.cpp \ node/coin.cpp \ node/connection_types.cpp \ node/context.cpp \ diff --git a/src/checkqueue.h b/src/checkqueue.h index bead6f0c6f..c4a64444e9 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -199,11 +199,12 @@ public: WITH_LOCK(m_mutex, m_request_stop = false); } + bool HasThreads() const { return !m_worker_threads.empty(); } + ~CCheckQueue() { assert(m_worker_threads.empty()); } - }; /** diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 85a7bbcb53..59b79609dd 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -30,7 +30,7 @@ void inline Round(uint64_t a, uint64_t b, uint64_t c, uint64_t& d, uint64_t e, u h = t1 + t2; } -/** Initialize SHA-256 state. */ +/** Initialize SHA-512 state. */ void inline Initialize(uint64_t* s) { s[0] = 0x6a09e667f3bcc908ull; diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 0e582629f7..f255834830 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -81,6 +81,9 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str for (const auto& entry : input.hd_keypaths) { if (parsed_m_fingerprint == MakeUCharSpan(entry.second.fingerprint)) return true; } + for (const auto& entry : input.m_tap_bip32_paths) { + if (parsed_m_fingerprint == MakeUCharSpan(entry.second.second.fingerprint)) return true; + } return false; }; diff --git a/src/init.cpp b/src/init.cpp index cf4102e6a7..24659de3df 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -40,6 +40,7 @@ #include <node/blockstorage.h> #include <node/caches.h> #include <node/chainstate.h> +#include <node/chainstatemanager_args.h> #include <node/context.h> #include <node/interface_ui.h> #include <node/mempool_args.h> @@ -554,7 +555,10 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_BYTES >> 20), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-maxtipage=<n>", + strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", + Ticks<std::chrono::seconds>(DEFAULT_MAX_TIP_AGE)), + ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-printpriority", strprintf("Log transaction fee rate in " + CURRENCY_UNIT + "/kvB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); @@ -930,21 +934,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb init::SetLoggingCategories(args); init::SetLoggingLevel(args); - fCheckBlockIndex = args.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); - fCheckpointsEnabled = args.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); - - hashAssumeValid = uint256S(args.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); - - if (args.IsArgSet("-minimumchainwork")) { - const std::string minChainWorkStr = args.GetArg("-minimumchainwork", ""); - if (!IsHexNumber(minChainWorkStr)) { - return InitError(strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), minChainWorkStr)); - } - nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); - } else { - nMinimumChainWork = UintToArith256(chainparams.GetConsensus().nMinimumChainWork); - } - // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg = args.GetIntArg("-prune", 0); if (nPruneArg < 0) { @@ -995,8 +984,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - nMaxTipAge = args.GetIntArg("-maxtipage", DEFAULT_MAX_TIP_AGE); - if (args.GetBoolArg("-reindex-chainstate", false)) { // indexes that must be deactivated to prevent index corruption, see #24630 if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { @@ -1044,6 +1031,16 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb } #endif // USE_SYSCALL_SANDBOX + // Also report errors from parsing before daemonization + { + ChainstateManager::Options chainman_opts_dummy{ + .chainparams = chainparams, + }; + if (const auto error{ApplyArgsManOptions(args, chainman_opts_dummy)}) { + return InitError(*error); + } + } + return true; } @@ -1146,7 +1143,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Script verification uses %d additional threads\n", script_threads); if (script_threads >= 1) { - g_parallel_script_checks = true; StartScriptCheckWorkerThreads(script_threads); } @@ -1435,6 +1431,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); + ChainstateManager::Options chainman_opts{ + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, + }; + Assert(!ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1471,10 +1472,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) for (bool fLoaded = false; !fLoaded && !ShutdownRequested();) { node.mempool = std::make_unique<CTxMemPool>(mempool_opts); - const ChainstateManager::Options chainman_opts{ - .chainparams = chainparams, - .adjusted_time_callback = GetAdjustedTime, - }; node.chainman = std::make_unique<ChainstateManager>(chainman_opts); ChainstateManager& chainman = *node.chainman; diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 5fc0e540a9..7a3d88b18f 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_INTERFACES_CHAIN_H #define BITCOIN_INTERFACES_CHAIN_H +#include <blockfilter.h> #include <primitives/transaction.h> // For CTransactionRef #include <util/settings.h> // For util::SettingsValue @@ -143,6 +144,13 @@ public: //! or one of its ancestors. virtual std::optional<int> findLocatorFork(const CBlockLocator& locator) = 0; + //! Returns whether a block filter index is available. + virtual bool hasBlockFilterIndex(BlockFilterType filter_type) = 0; + + //! Returns whether any of the elements match the block via a BIP 157 block filter + //! or std::nullopt if the block filter for this block couldn't be found. + virtual std::optional<bool> blockFilterMatchesAny(BlockFilterType filter_type, const uint256& block_hash, const GCSFilter::ElementSet& filter_set) = 0; + //! Return whether node has the block and optionally return block metadata //! or contents. virtual bool findBlock(const uint256& hash, const FoundBlock& block={}) = 0; diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 520d0e8e75..020ae24c11 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -5,13 +5,19 @@ #ifndef BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H #define BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H +#include <arith_uint256.h> +#include <uint256.h> #include <util/time.h> #include <cstdint> #include <functional> +#include <optional> class CChainParams; +static constexpr bool DEFAULT_CHECKPOINTS_ENABLED{true}; +static constexpr auto DEFAULT_MAX_TIP_AGE{24h}; + namespace kernel { /** @@ -22,6 +28,14 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; const std::function<NodeClock::time_point()> adjusted_time_callback{nullptr}; + std::optional<bool> check_block_index{}; + bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; + //! If set, it will override the minimum work we will assume exists on some valid chain. + std::optional<arith_uint256> minimum_chain_work; + //! If set, it will override the block hash whose ancestors we will assume to have valid scripts without checking them. + std::optional<uint256> assumed_valid_block; + //! If the tip is older than this, the node is considered to be in initial block download. + std::chrono::seconds max_tip_age{DEFAULT_MAX_TIP_AGE}; }; } // namespace kernel diff --git a/src/logging.cpp b/src/logging.cpp index c95c0b7e37..ed0c2a56a5 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -181,6 +181,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::UTIL, "util"}, {BCLog::BLOCKSTORE, "blockstorage"}, {BCLog::TXRECONCILIATION, "txreconciliation"}, + {BCLog::SCAN, "scan"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -283,6 +284,8 @@ std::string LogCategoryToStr(BCLog::LogFlags category) return "blockstorage"; case BCLog::LogFlags::TXRECONCILIATION: return "txreconciliation"; + case BCLog::LogFlags::SCAN: + return "scan"; case BCLog::LogFlags::ALL: return "all"; } diff --git a/src/logging.h b/src/logging.h index 5ee6665c76..14a0f08f8d 100644 --- a/src/logging.h +++ b/src/logging.h @@ -67,6 +67,7 @@ namespace BCLog { UTIL = (1 << 25), BLOCKSTORE = (1 << 26), TXRECONCILIATION = (1 << 27), + SCAN = (1 << 28), ALL = ~(uint32_t)0, }; enum class Level { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 3f344005bc..363f2fde71 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1291,7 +1291,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co // Make sure pindexBestKnownBlock is up to date, we'll need it. ProcessBlockAvailability(peer.m_id); - if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < m_chainman.ActiveChain().Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < m_chainman.ActiveChain().Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < m_chainman.MinimumChainWork()) { // This peer has nothing interesting. return; } @@ -2392,7 +2392,7 @@ arith_uint256 PeerManagerImpl::GetAntiDoSWorkThreshold() // near our tip. near_chaintip_work = tip->nChainWork - std::min<arith_uint256>(144*GetBlockProof(*tip), tip->nChainWork); } - return std::max(near_chaintip_work, arith_uint256(nMinimumChainWork)); + return std::max(near_chaintip_work, m_chainman.MinimumChainWork()); } /** @@ -2710,14 +2710,14 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { // If the peer has no more headers to give us, then we know we have // their tip. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < m_chainman.MinimumChainWork()) { // This peer has too little work on their headers chain to help // us sync -- disconnect if it is an outbound disconnection // candidate. - // Note: We compare their tip to nMinimumChainWork (rather than + // Note: We compare their tip to the minumum chain work (rather than // m_chainman.ActiveChain().Tip()) because we won't start block download // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, + // the minimum chain work, even if a peer has a chain past our tip, // as an anti-DoS measure. if (pfrom.IsOutboundOrBlockRelayConn()) { LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); @@ -3901,12 +3901,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Note that if we were to be on a chain that forks from the checkpointed // chain, then serving those headers to a peer that has seen the // checkpointed chain would cause that peer to disconnect us. Requiring - // that our chainwork exceed nMinimumChainWork is a protection against + // that our chainwork exceed the mimimum chain work is a protection against // being fed a bogus chain when we started up for the first time and // getting partitioned off the honest network for serving that chain to // others. if (m_chainman.ActiveTip() == nullptr || - (m_chainman.ActiveTip()->nChainWork < nMinimumChainWork && !pfrom.HasPermission(NetPermissionFlags::Download))) { + (m_chainman.ActiveTip()->nChainWork < m_chainman.MinimumChainWork() && !pfrom.HasPermission(NetPermissionFlags::Download))) { LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); // Just respond with an empty headers message, to tell the peer to // go away but not treat us as unresponsive. @@ -4370,7 +4370,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // (eg disk space). Because we only try to reconstruct blocks when // we're close to caught up (via the CanDirectFetch() requirement // above, combined with the behavior of not requesting blocks until - // we have a chain with at least nMinimumChainWork), and we ignore + // we have a chain with at least the minimum chain work), and we ignore // compact blocks with less work than our tip, it is safe to treat // reconstructed compact blocks as having been requested. ProcessBlock(pfrom, pblock, /*force_processing=*/true, /*min_pow_checked=*/true); @@ -5236,7 +5236,7 @@ void PeerManagerImpl::MaybeSendSendHeaders(CNode& node, Peer& peer) LOCK(cs_main); CNodeState &state = *State(node.GetId()); if (state.pindexBestKnownBlock != nullptr && - state.pindexBestKnownBlock->nChainWork > nMinimumChainWork) { + state.pindexBestKnownBlock->nChainWork > m_chainman.MinimumChainWork()) { // Tell our peer we prefer to receive headers rather than inv's // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 26af523491..60acb614b4 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -4,9 +4,11 @@ #include <node/chainstate.h> +#include <arith_uint256.h> #include <chain.h> #include <coins.h> #include <consensus/params.h> +#include <logging.h> #include <node/blockstorage.h> #include <node/caches.h> #include <sync.h> @@ -21,6 +23,7 @@ #include <algorithm> #include <atomic> #include <cassert> +#include <limits> #include <memory> #include <vector> @@ -32,13 +35,13 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; - if (!hashAssumeValid.IsNull()) { - LogPrintf("Assuming ancestors of block %s have valid signatures.\n", hashAssumeValid.GetHex()); + if (!chainman.AssumedValidBlock().IsNull()) { + LogPrintf("Assuming ancestors of block %s have valid signatures.\n", chainman.AssumedValidBlock().GetHex()); } else { LogPrintf("Validating signatures for all blocks.\n"); } - LogPrintf("Setting nMinimumChainWork=%s\n", nMinimumChainWork.GetHex()); - if (nMinimumChainWork < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { + LogPrintf("Setting nMinimumChainWork=%s\n", chainman.MinimumChainWork().GetHex()); + if (chainman.MinimumChainWork() < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainman.GetConsensus().nMinimumChainWork.GetHex()); } if (nPruneTarget == std::numeric_limits<uint64_t>::max()) { diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp new file mode 100644 index 0000000000..b0d929626b --- /dev/null +++ b/src/node/chainstatemanager_args.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/chainstatemanager_args.h> + +#include <arith_uint256.h> +#include <tinyformat.h> +#include <uint256.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <util/translation.h> +#include <validation.h> + +#include <chrono> +#include <optional> +#include <string> + +namespace node { +std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) +{ + if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + + if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; + + if (auto value{args.GetArg("-minimumchainwork")}) { + if (!IsHexNumber(*value)) { + return strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value); + } + opts.minimum_chain_work = UintToArith256(uint256S(*value)); + } + + if (auto value{args.GetArg("-assumevalid")}) opts.assumed_valid_block = uint256S(*value); + + if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value}; + + return std::nullopt; +} +} // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h new file mode 100644 index 0000000000..6c46b998f2 --- /dev/null +++ b/src/node/chainstatemanager_args.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H +#define BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H + +#include <validation.h> + +#include <optional> + +class ArgsManager; +struct bilingual_str; + +namespace node { +std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); +} // namespace node + +#endif // BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 8a0011a629..979c625463 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -4,10 +4,12 @@ #include <addrdb.h> #include <banman.h> +#include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <deploymentstatus.h> #include <external_signer.h> +#include <index/blockfilterindex.h> #include <init.h> #include <interfaces/chain.h> #include <interfaces/handler.h> @@ -536,6 +538,20 @@ public: } return std::nullopt; } + bool hasBlockFilterIndex(BlockFilterType filter_type) override + { + return GetBlockFilterIndex(filter_type) != nullptr; + } + std::optional<bool> blockFilterMatchesAny(BlockFilterType filter_type, const uint256& block_hash, const GCSFilter::ElementSet& filter_set) override + { + const BlockFilterIndex* block_filter_index{GetBlockFilterIndex(filter_type)}; + if (!block_filter_index) return std::nullopt; + + BlockFilter filter; + const CBlockIndex* index{WITH_LOCK(::cs_main, return chainman().m_blockman.LookupBlockIndex(block_hash))}; + if (index == nullptr || !block_filter_index->LookupFilter(index, filter)) return std::nullopt; + return filter.GetFilter().MatchAny(filter_set); + } bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 894a401e56..f522d78be4 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -410,7 +410,7 @@ void BitcoinGUI::createActions() connect(action, &QAction::triggered, [this, path] { auto activity = new OpenWalletActivity(m_wallet_controller, this); - connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet); + connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet, Qt::QueuedConnection); activity->open(path); }); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f8ba822f54..bc1028aec3 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -459,6 +459,12 @@ static RPCHelpMan getblockfrompeer() throw JSONRPCError(RPC_MISC_ERROR, "Block header missing"); } + // Fetching blocks before the node has syncing past their height can prevent block files from + // being pruned, so we avoid it if the node is in prune mode. + if (node::fPruneMode && index->nHeight > WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()->nHeight)) { + throw JSONRPCError(RPC_MISC_ERROR, "In prune mode, only blocks that the node has already synced previously can be fetched from a peer"); + } + const bool block_has_data = WITH_LOCK(::cs_main, return index->nStatus & BLOCK_HAVE_DATA); if (block_has_data) { throw JSONRPCError(RPC_MISC_ERROR, "Block already downloaded"); diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index 744f809814..a980c609e8 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -273,7 +273,7 @@ static RPCHelpMan deriveaddresses() UniValue addresses(UniValue::VARR); - for (int i = range_begin; i <= range_end; ++i) { + for (int64_t i = range_begin; i <= range_end; ++i) { FlatSigningProvider provider; std::vector<CScript> scripts; if (!desc->Expand(i, key_provider, scripts, provider)) { diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 9f29342b10..0d0db176f9 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -146,7 +146,6 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve Assert(InitScriptExecutionCache(validation_cache_sizes.script_execution_cache_bytes)); m_node.chain = interfaces::MakeChain(m_node); - fCheckBlockIndex = true; static bool noui_connected = false; if (!noui_connected) { noui_connect(); @@ -181,14 +180,13 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .adjusted_time_callback = GetAdjustedTime, + .check_block_index = true, }; m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(m_cache_sizes.block_tree_db, true); - // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; StartScriptCheckWorkerThreads(script_check_threads); - g_parallel_script_checks = true; } ChainTestingSetup::~ChainTestingSetup() diff --git a/src/validation.cpp b/src/validation.cpp index 37e68cfe4a..debdc2ae74 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -120,13 +120,6 @@ RecursiveMutex cs_main; GlobalMutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; -bool g_parallel_script_checks{false}; -bool fCheckBlockIndex = false; -bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; - -uint256 hashAssumeValid; -arith_uint256 nMinimumChainWork; const CBlockIndex* Chainstate::FindForkInGlobalIndex(const CBlockLocator& locator) const { @@ -1545,10 +1538,12 @@ bool Chainstate::IsInitialBlockDownload() const return true; if (m_chain.Tip() == nullptr) return true; - if (m_chain.Tip()->nChainWork < nMinimumChainWork) + if (m_chain.Tip()->nChainWork < m_chainman.MinimumChainWork()) { return true; - if (m_chain.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) + } + if (m_chain.Tip()->Time() < NodeClock::now() - m_chainman.m_options.max_tip_age) { return true; + } LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); m_cached_finished_ibd.store(true, std::memory_order_relaxed); return false; @@ -1994,6 +1989,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, uint256 block_hash{block.GetHash()}; assert(*pindex->phashBlock == block_hash); + const bool parallel_script_checks{scriptcheckqueue.HasThreads()}; const auto time_start{SteadyClock::now()}; const CChainParams& params{m_chainman.GetParams()}; @@ -2036,17 +2032,17 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, } bool fScriptChecks = true; - if (!hashAssumeValid.IsNull()) { + if (!m_chainman.AssumedValidBlock().IsNull()) { // We've been configured with the hash of a block which has been externally verified to have a valid history. // A suitable default value is included with the software and updated from time to time. Because validity // relative to a piece of software is an objective fact these defaults can be easily reviewed. // This setting doesn't force the selection of any particular chain but makes validating some faster by // effectively caching the result of part of the verification. - BlockMap::const_iterator it = m_blockman.m_block_index.find(hashAssumeValid); + BlockMap::const_iterator it{m_blockman.m_block_index.find(m_chainman.AssumedValidBlock())}; if (it != m_blockman.m_block_index.end()) { if (it->second.GetAncestor(pindex->nHeight) == pindex && m_chainman.m_best_header->GetAncestor(pindex->nHeight) == pindex && - m_chainman.m_best_header->nChainWork >= nMinimumChainWork) { + m_chainman.m_best_header->nChainWork >= m_chainman.MinimumChainWork()) { // This block is a member of the assumed verified chain and an ancestor of the best header. // Script verification is skipped when connecting blocks under the // assumevalid block. Assuming the assumevalid block is valid this @@ -2059,7 +2055,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // it hard to hide the implication of the demand. This also avoids having release candidates // that are hardly doing any signature verification at all in testing without having to // artificially set the default assumed verified block further back. - // The test against nMinimumChainWork prevents the skipping when denied access to any chain at + // The test against the minimum chain work prevents the skipping when denied access to any chain at // least as good as the expected chain. fScriptChecks = (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); } @@ -2183,7 +2179,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // in multiple threads). Preallocate the vector size so a new allocation // doesn't invalidate pointers into the vector, and keep txsdata in scope // for as long as `control`. - CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); + CCheckQueueControl<CScriptCheck> control(fScriptChecks && parallel_script_checks ? &scriptcheckqueue : nullptr); std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; @@ -2242,7 +2238,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ TxValidationState tx_state; - if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], parallel_script_checks ? &vChecks : nullptr)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); @@ -3532,7 +3528,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "bad-diffbits", "incorrect proof of work"); // Check against checkpoints - if (fCheckpointsEnabled) { + if (chainman.m_options.checkpoints_enabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). @@ -3846,7 +3842,7 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // If our tip is behind, a peer could try to send us // low-work blocks on a fake chain that we would never // request; don't process these. - if (pindex->nChainWork < nMinimumChainWork) return true; + if (pindex->nChainWork < m_chainman.MinimumChainWork()) return true; } const CChainParams& params{m_chainman.GetParams()}; @@ -4517,7 +4513,7 @@ void Chainstate::LoadExternalBlockFile( void Chainstate::CheckBlockIndex() { - if (!fCheckBlockIndex) { + if (!m_chainman.ShouldCheckBlockIndex()) { return; } @@ -5251,6 +5247,22 @@ void ChainstateManager::ResetChainstates() m_active_chainstate = nullptr; } +/** + * Apply default chain params to nullopt members. + * This helps to avoid coding errors around the accidental use of the compare + * operators that accept nullopt, thus ignoring the intended default value. + */ +static ChainstateManager::Options&& Flatten(ChainstateManager::Options&& opts) +{ + if (!opts.check_block_index.has_value()) opts.check_block_index = opts.chainparams.DefaultConsistencyChecks(); + if (!opts.minimum_chain_work.has_value()) opts.minimum_chain_work = UintToArith256(opts.chainparams.GetConsensus().nMinimumChainWork); + if (!opts.assumed_valid_block.has_value()) opts.assumed_valid_block = opts.chainparams.GetConsensus().defaultAssumeValid; + Assert(opts.adjusted_time_callback); + return std::move(opts); +} + +ChainstateManager::ChainstateManager(Options options) : m_options{Flatten(std::move(options))} {} + ChainstateManager::~ChainstateManager() { LOCK(::cs_main); diff --git a/src/validation.h b/src/validation.h index fb6d59f92e..b8151dc1fc 100644 --- a/src/validation.h +++ b/src/validation.h @@ -63,8 +63,6 @@ struct Params; static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; -static const bool DEFAULT_CHECKPOINTS_ENABLED = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ @@ -93,20 +91,6 @@ extern GlobalMutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; /** Used to notify getblocktemplate RPC of new tips. */ extern uint256 g_best_block; -/** Whether there are dedicated script-checking threads running. - * False indicates all script checking is done on the main threadMessageHandler thread. - */ -extern bool g_parallel_script_checks; -extern bool fCheckBlockIndex; -extern bool fCheckpointsEnabled; -/** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ -extern int64_t nMaxTipAge; - -/** Block hash whose ancestors we will assume to have valid scripts without checking them. */ -extern uint256 hashAssumeValid; - -/** Minimum work we will assume exists on some valid chain. */ -extern arith_uint256 nMinimumChainWork; /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; @@ -698,7 +682,7 @@ public: /** * Make various assertions about the state of the block index. * - * By default this only executes fully when using the Regtest chain; see: fCheckBlockIndex. + * By default this only executes fully when using the Regtest chain; see: m_options.check_block_index. */ void CheckBlockIndex(); @@ -863,13 +847,13 @@ private: public: using Options = kernel::ChainstateManagerOpts; - explicit ChainstateManager(Options options) : m_options{std::move(options)} - { - Assert(m_options.adjusted_time_callback); - } + explicit ChainstateManager(Options options); const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } + bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } + const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } /** * Alias for ::cs_main. diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 1206a428fc..1d2d7d2a10 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -1589,7 +1589,8 @@ RPCHelpMan importdescriptors() return RPCHelpMan{"importdescriptors", "\nImport descriptors. This will trigger a rescan of the blockchain based on the earliest timestamp of all descriptors being imported. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if using an early timestamp; during that time, other rpc calls\n" - "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n", + "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan is significantly faster if block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", { @@ -1887,7 +1888,9 @@ RPCHelpMan restorewallet() { return RPCHelpMan{ "restorewallet", - "\nRestore and loads a wallet from backup.\n", + "\nRestore and loads a wallet from backup.\n" + "\nThe rescan is significantly faster if a descriptor wallet is restored" + "\nand block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name that will be applied to the restored wallet"}, {"backup_file", RPCArg::Type::STR, RPCArg::Optional::NO, "The backup file that will be used to restore the wallet."}, diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 0e13e4756b..0fe4ed703d 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -839,7 +839,9 @@ RPCHelpMan rescanblockchain() { return RPCHelpMan{"rescanblockchain", "\nRescan the local blockchain for wallet related transactions.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Use \"getwalletinfo\" to query the scanning progress.\n" + "The rescan is significantly faster when used on a descriptor wallet\n" + "and block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "block height where the rescan should start"}, {"stop_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 39afb79600..4c534d64ec 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -2645,16 +2645,26 @@ const WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const const std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys() const { + return GetScriptPubKeys(0); +} + +const std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys(int32_t minimum_index) const +{ LOCK(cs_desc_man); std::unordered_set<CScript, SaltedSipHasher> script_pub_keys; script_pub_keys.reserve(m_map_script_pub_keys.size()); - for (auto const& script_pub_key: m_map_script_pub_keys) { - script_pub_keys.insert(script_pub_key.first); + for (auto const& [script_pub_key, index] : m_map_script_pub_keys) { + if (index >= minimum_index) script_pub_keys.insert(script_pub_key); } return script_pub_keys; } +int32_t DescriptorScriptPubKeyMan::GetEndRange() const +{ + return m_max_cached_index + 1; +} + bool DescriptorScriptPubKeyMan::GetDescriptorString(std::string& out, const bool priv) const { LOCK(cs_desc_man); diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 3ab489c374..eb77015956 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -643,6 +643,8 @@ public: const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const override; + const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys(int32_t minimum_index) const; + int32_t GetEndRange() const; bool GetDescriptorString(std::string& out, const bool priv) const; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e2c8f6eda3..431e970edc 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5,6 +5,7 @@ #include <wallet/wallet.h> +#include <blockfilter.h> #include <chain.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -261,6 +262,64 @@ std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::s return nullptr; } } + +class FastWalletRescanFilter +{ +public: + FastWalletRescanFilter(const CWallet& wallet) : m_wallet(wallet) + { + // fast rescanning via block filters is only supported by descriptor wallets right now + assert(!m_wallet.IsLegacy()); + + // create initial filter with scripts from all ScriptPubKeyMans + for (auto spkm : m_wallet.GetAllScriptPubKeyMans()) { + auto desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(spkm)}; + assert(desc_spkm != nullptr); + AddScriptPubKeys(desc_spkm); + // save each range descriptor's end for possible future filter updates + if (desc_spkm->IsHDEnabled()) { + m_last_range_ends.emplace(desc_spkm->GetID(), desc_spkm->GetEndRange()); + } + } + } + + void UpdateIfNeeded() + { + // repopulate filter with new scripts if top-up has happened since last iteration + for (const auto& [desc_spkm_id, last_range_end] : m_last_range_ends) { + auto desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(m_wallet.GetScriptPubKeyMan(desc_spkm_id))}; + assert(desc_spkm != nullptr); + int32_t current_range_end{desc_spkm->GetEndRange()}; + if (current_range_end > last_range_end) { + AddScriptPubKeys(desc_spkm, last_range_end); + m_last_range_ends.at(desc_spkm->GetID()) = current_range_end; + } + } + } + + std::optional<bool> MatchesBlock(const uint256& block_hash) const + { + return m_wallet.chain().blockFilterMatchesAny(BlockFilterType::BASIC, block_hash, m_filter_set); + } + +private: + const CWallet& m_wallet; + /** Map for keeping track of each range descriptor's last seen end range. + * This information is used to detect whether new addresses were derived + * (that is, if the current end range is larger than the saved end range) + * after processing a block and hence a filter set update is needed to + * take possible keypool top-ups into account. + */ + std::map<uint256, int32_t> m_last_range_ends; + GCSFilter::ElementSet m_filter_set; + + void AddScriptPubKeys(const DescriptorScriptPubKeyMan* desc_spkm, int32_t last_range_end = 0) + { + for (const auto& script_pub_key : desc_spkm->GetScriptPubKeys(last_range_end)) { + m_filter_set.emplace(script_pub_key.begin(), script_pub_key.end()); + } + } +}; } // namespace std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) @@ -1755,7 +1814,11 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc uint256 block_hash = start_block; ScanResult result; - WalletLogPrintf("Rescan started from block %s...\n", start_block.ToString()); + std::unique_ptr<FastWalletRescanFilter> fast_rescan_filter; + if (!IsLegacy() && chain().hasBlockFilterIndex(BlockFilterType::BASIC)) fast_rescan_filter = std::make_unique<FastWalletRescanFilter>(*this); + + WalletLogPrintf("Rescan started from block %s... (%s)\n", start_block.ToString(), + fast_rescan_filter ? "fast variant using block filters" : "slow variant inspecting all blocks"); fAbortRescan = false; ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if rescan required on startup (e.g. due to corruption) @@ -1782,9 +1845,22 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } - // Read block data - CBlock block; - chain().findBlock(block_hash, FoundBlock().data(block)); + bool fetch_block{true}; + if (fast_rescan_filter) { + fast_rescan_filter->UpdateIfNeeded(); + auto matches_block{fast_rescan_filter->MatchesBlock(block_hash)}; + if (matches_block.has_value()) { + if (*matches_block) { + LogPrint(BCLog::SCAN, "Fast rescan: inspect block %d [%s] (filter matched)\n", block_height, block_hash.ToString()); + } else { + result.last_scanned_block = block_hash; + result.last_scanned_height = block_height; + fetch_block = false; + } + } else { + LogPrint(BCLog::SCAN, "Fast rescan: inspect block %d [%s] (WARNING: block filter not found!)\n", block_height, block_hash.ToString()); + } + } // Find next block separately from reading data above, because reading // is slow and there might be a reorg while it is read. @@ -1793,35 +1869,41 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc uint256 next_block_hash; chain().findBlock(block_hash, FoundBlock().inActiveChain(block_still_active).nextBlock(FoundBlock().inActiveChain(next_block).hash(next_block_hash))); - if (!block.IsNull()) { - LOCK(cs_wallet); - if (!block_still_active) { - // Abort scan if current block is no longer active, to prevent - // marking transactions as coming from the wrong block. - result.last_failed_block = block_hash; - result.status = ScanResult::FAILURE; - break; - } - for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], TxStateConfirmed{block_hash, block_height, static_cast<int>(posInBlock)}, fUpdate, /*rescanning_old_block=*/true); - } - // scan succeeded, record block as most recent successfully scanned - result.last_scanned_block = block_hash; - result.last_scanned_height = block_height; + if (fetch_block) { + // Read block data + CBlock block; + chain().findBlock(block_hash, FoundBlock().data(block)); + + if (!block.IsNull()) { + LOCK(cs_wallet); + if (!block_still_active) { + // Abort scan if current block is no longer active, to prevent + // marking transactions as coming from the wrong block. + result.last_failed_block = block_hash; + result.status = ScanResult::FAILURE; + break; + } + for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { + SyncTransaction(block.vtx[posInBlock], TxStateConfirmed{block_hash, block_height, static_cast<int>(posInBlock)}, fUpdate, /*rescanning_old_block=*/true); + } + // scan succeeded, record block as most recent successfully scanned + result.last_scanned_block = block_hash; + result.last_scanned_height = block_height; - if (save_progress && next_interval) { - CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); + if (save_progress && next_interval) { + CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); - if (!loc.IsNull()) { - WalletLogPrintf("Saving scan progress %d.\n", block_height); - WalletBatch batch(GetDatabase()); - batch.WriteBestBlock(loc); + if (!loc.IsNull()) { + WalletLogPrintf("Saving scan progress %d.\n", block_height); + WalletBatch batch(GetDatabase()); + batch.WriteBestBlock(loc); + } } + } else { + // could not scan block, keep scanning but record this block as the most recent failure + result.last_failed_block = block_hash; + result.status = ScanResult::FAILURE; } - } else { - // could not scan block, keep scanning but record this block as the most recent failure - result.last_failed_block = block_hash; - result.status = ScanResult::FAILURE; } if (max_height && block_height >= *max_height) { break; diff --git a/test/functional/feature_maxtipage.py b/test/functional/feature_maxtipage.py index 87f9d6962d..ddc2102542 100755 --- a/test/functional/feature_maxtipage.py +++ b/test/functional/feature_maxtipage.py @@ -2,10 +2,10 @@ # Copyright (c) 2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test logic for setting nMaxTipAge on command line. +"""Test logic for setting -maxtipage on command line. Nodes don't consider themselves out of "initial block download" as long as -their best known block header time is more than nMaxTipAge in the past. +their best known block header time is more than -maxtipage in the past. """ import time diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index c5a8f7b1e9..6699914249 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -27,12 +27,15 @@ def getdescriptors(args): "receive": [ "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs" + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs", + "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/0/*)#sng9rd4t" ], "internal": [ "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg" + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg", + "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/1/*)#p8dy7c9n" + ] })) @@ -44,7 +47,8 @@ def displayaddress(args): return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) expected_desc = [ - "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r" + "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r", + "tr([00000001/86'/1'/0'/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#4vdj9jqk", ] if args.desc not in expected_desc: return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) diff --git a/test/functional/p2p_sendtxrcncl.py b/test/functional/p2p_sendtxrcncl.py index f4c5dd4586..14cd815a30 100755 --- a/test/functional/p2p_sendtxrcncl.py +++ b/test/functional/p2p_sendtxrcncl.py @@ -107,37 +107,39 @@ class SendTxRcnclTest(BitcoinTestFramework): peer.send_message(create_sendtxrcncl_msg()) self.wait_until(lambda : "sendtxrcncl" in self.nodes[0].getpeerinfo()[-1]["bytesrecv_per_msg"]) self.log.info('second SENDTXRCNCL triggers a disconnect') - peer.send_message(create_sendtxrcncl_msg()) - peer.wait_for_disconnect() + with self.nodes[0].assert_debug_log(["(sendtxrcncl received from already registered peer); disconnecting"]): + peer.send_message(create_sendtxrcncl_msg()) + peer.wait_for_disconnect() self.log.info('SENDTXRCNCL with initiator=responder=0 triggers a disconnect') sendtxrcncl_no_role = create_sendtxrcncl_msg() sendtxrcncl_no_role.initiator = False sendtxrcncl_no_role.responder = False peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) - peer.send_message(sendtxrcncl_no_role) - peer.wait_for_disconnect() + with self.nodes[0].assert_debug_log(["txreconciliation protocol violation"]): + peer.send_message(sendtxrcncl_no_role) + peer.wait_for_disconnect() self.log.info('SENDTXRCNCL with initiator=0 and responder=1 from inbound triggers a disconnect') sendtxrcncl_wrong_role = create_sendtxrcncl_msg(initiator=False) peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) - peer.send_message(sendtxrcncl_wrong_role) - peer.wait_for_disconnect() + with self.nodes[0].assert_debug_log(["txreconciliation protocol violation"]): + peer.send_message(sendtxrcncl_wrong_role) + peer.wait_for_disconnect() self.log.info('SENDTXRCNCL with version=0 triggers a disconnect') sendtxrcncl_low_version = create_sendtxrcncl_msg() sendtxrcncl_low_version.version = 0 peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) - peer.send_message(sendtxrcncl_low_version) - peer.wait_for_disconnect() + with self.nodes[0].assert_debug_log(["txreconciliation protocol violation"]): + peer.send_message(sendtxrcncl_low_version) + peer.wait_for_disconnect() self.log.info('sending SENDTXRCNCL after sending VERACK triggers a disconnect') - # We use PeerNoVerack even though verack is sent right after, to make sure it was actually - # sent before sendtxrcncl is sent. - peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) - peer.send_and_ping(msg_verack()) - peer.send_message(create_sendtxrcncl_msg()) - peer.wait_for_disconnect() + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(["sendtxrcncl received after verack"]): + peer.send_message(create_sendtxrcncl_msg()) + peer.wait_for_disconnect() self.log.info('SENDTXRCNCL without WTXIDRELAY is ignored (recon state is erased after VERACK)') peer = self.nodes[0].add_p2p_connection(PeerNoVerack(wtxidrelay=False), send_version=True, wait_for_verack=False) @@ -164,15 +166,17 @@ class SendTxRcnclTest(BitcoinTestFramework): self.log.info('SENDTXRCNCL if block-relay-only triggers a disconnect') peer = self.nodes[0].add_outbound_p2p_connection( PeerNoVerack(), wait_for_verack=False, p2p_idx=3, connection_type="block-relay-only") - peer.send_message(create_sendtxrcncl_msg(initiator=False)) - peer.wait_for_disconnect() + with self.nodes[0].assert_debug_log(["we indicated no tx relay; disconnecting"]): + peer.send_message(create_sendtxrcncl_msg(initiator=False)) + peer.wait_for_disconnect() self.log.info('SENDTXRCNCL with initiator=1 and responder=0 from outbound triggers a disconnect') sendtxrcncl_wrong_role = create_sendtxrcncl_msg(initiator=True) peer = self.nodes[0].add_outbound_p2p_connection( - P2PInterface(), wait_for_verack=False, p2p_idx=4, connection_type="outbound-full-relay") - peer.send_message(sendtxrcncl_wrong_role) - peer.wait_for_disconnect() + PeerNoVerack(), wait_for_verack=False, p2p_idx=4, connection_type="outbound-full-relay") + with self.nodes[0].assert_debug_log(["txreconciliation protocol violation"]): + peer.send_message(sendtxrcncl_wrong_role) + peer.wait_for_disconnect() self.log.info('SENDTXRCNCL not sent if -txreconciliation flag is not set') self.restart_node(0, []) diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py index 42d7d59d56..a69326736d 100755 --- a/test/functional/rpc_deriveaddresses.py +++ b/test/functional/rpc_deriveaddresses.py @@ -44,6 +44,13 @@ class DeriveaddressesTest(BitcoinTestFramework): combo_descriptor = descsum_create("combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)") assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) + # Before #26275, bitcoind would crash when deriveaddresses was + # called with derivation index 2147483647, which is the maximum + # positive value of a signed int32, and - currently - the + # maximum value that the deriveaddresses bitcoin RPC call + # accepts as derivation index. + assert_equal(self.nodes[0].deriveaddresses(descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [2147483647, 2147483647]), ["bcrt1qtzs23vgzpreks5gtygwxf8tv5rldxvvsyfpdkg"]) + hardened_without_privkey_descriptor = descsum_create("wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)") assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index 278a343b2b..fd4d1992eb 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -5,7 +5,12 @@ """Test the getblockfrompeer RPC.""" from test_framework.authproxy import JSONRPCException -from test_framework.messages import NODE_WITNESS +from test_framework.messages import ( + CBlock, + from_hex, + msg_headers, + NODE_WITNESS, +) from test_framework.p2p import ( P2P_SERVICES, P2PInterface, @@ -16,6 +21,7 @@ from test_framework.util import ( assert_raises_rpc_error, ) + class GetBlockFromPeerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -81,6 +87,30 @@ class GetBlockFromPeerTest(BitcoinTestFramework): self.log.info("Don't fetch blocks we already have") assert_raises_rpc_error(-1, "Block already downloaded", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id) + self.log.info("Don't fetch blocks while the node has not synced past it yet") + # For this test we need node 1 in prune mode and as a side effect this also disconnects + # the nodes which is also necessary for the rest of the test. + self.restart_node(1, ["-prune=550"]) + + # Generate a block on the disconnected node that the pruning node is not connected to + blockhash = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[0] + block_hex = self.nodes[0].getblock(blockhash=blockhash, verbosity=0) + block = from_hex(CBlock(), block_hex) + + # Connect a P2PInterface to the pruning node and have it submit only the header of the + # block that the pruning node has not seen + node1_interface = self.nodes[1].add_p2p_connection(P2PInterface()) + node1_interface.send_message(msg_headers([block])) + + # Get the peer id of the P2PInterface from the pruning node + node1_peers = self.nodes[1].getpeerinfo() + assert_equal(len(node1_peers), 1) + node1_interface_id = node1_peers[0]["id"] + + # Trying to fetch this block from the P2PInterface should not be possible + error_msg = "In prune mode, only blocks that the node has already synced previously can be fetched from a peer" + assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id) + if __name__ == '__main__': GetBlockFromPeerTest().main() diff --git a/test/functional/rpc_scanblocks.py b/test/functional/rpc_scanblocks.py index 39f091fd1a..68c84b17a2 100755 --- a/test/functional/rpc_scanblocks.py +++ b/test/functional/rpc_scanblocks.py @@ -3,6 +3,10 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scanblocks RPC call.""" +from test_framework.blockfilter import ( + bip158_basic_element_hash, + bip158_relevant_scriptpubkeys, +) from test_framework.messages import COIN from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -71,6 +75,28 @@ class ScanblocksTest(BitcoinTestFramework): assert(blockhash in node.scanblocks( "start", [{"desc": f"pkh({parent_key}/*)", "range": [0, 100]}], height)['relevant_blocks']) + # check that false-positives are included in the result now; note that + # finding a false-positive at runtime would take too long, hence we simply + # use a pre-calculated one that collides with the regtest genesis block's + # coinbase output and verify that their BIP158 ranged hashes match + genesis_blockhash = node.getblockhash(0) + genesis_spks = bip158_relevant_scriptpubkeys(node, genesis_blockhash) + assert_equal(len(genesis_spks), 1) + genesis_coinbase_spk = list(genesis_spks)[0] + false_positive_spk = bytes.fromhex("001400000000000000000000000000000000000cadcb") + + genesis_coinbase_hash = bip158_basic_element_hash(genesis_coinbase_spk, 1, genesis_blockhash) + false_positive_hash = bip158_basic_element_hash(false_positive_spk, 1, genesis_blockhash) + assert_equal(genesis_coinbase_hash, false_positive_hash) + + assert(genesis_blockhash in node.scanblocks( + "start", [{"desc": f"raw({genesis_coinbase_spk.hex()})"}], 0, 0)['relevant_blocks']) + assert(genesis_blockhash in node.scanblocks( + "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks']) + + # TODO: after an "accurate" mode for scanblocks is implemented (e.g. PR #26325) + # check here that it filters out the false-positive + # test node with disabled blockfilterindex assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[1].scanblocks, "start", [f"addr({addr_1})"]) diff --git a/test/functional/test_framework/blockfilter.py b/test/functional/test_framework/blockfilter.py new file mode 100644 index 0000000000..a30e37ea5b --- /dev/null +++ b/test/functional/test_framework/blockfilter.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Helper routines relevant for compact block filters (BIP158). +""" +from .siphash import siphash + + +def bip158_basic_element_hash(script_pub_key, N, block_hash): + """ Calculates the ranged hash of a filter element as defined in BIP158: + + 'The first step in the filter construction is hashing the variable-sized + raw items in the set to the range [0, F), where F = N * M.' + + 'The items are first passed through the pseudorandom function SipHash, which takes a + 128-bit key k and a variable-sized byte vector and produces a uniformly random 64-bit + output. Implementations of this BIP MUST use the SipHash parameters c = 2 and d = 4.' + + 'The parameter k MUST be set to the first 16 bytes of the hash (in standard + little-endian representation) of the block for which the filter is constructed. This + ensures the key is deterministic while still varying from block to block.' + """ + M = 784931 + block_hash_bytes = bytes.fromhex(block_hash)[::-1] + k0 = int.from_bytes(block_hash_bytes[0:8], 'little') + k1 = int.from_bytes(block_hash_bytes[8:16], 'little') + return (siphash(k0, k1, script_pub_key) * (N * M)) >> 64 + + +def bip158_relevant_scriptpubkeys(node, block_hash): + """ Determines the basic filter relvant scriptPubKeys as defined in BIP158: + + 'A basic filter MUST contain exactly the following items for each transaction in a block: + - The previous output script (the script being spent) for each input, except for + the coinbase transaction. + - The scriptPubKey of each output, aside from all OP_RETURN output scripts.' + """ + spks = set() + for tx in node.getblock(blockhash=block_hash, verbosity=3)['tx']: + # gather prevout scripts + for i in tx['vin']: + if 'prevout' in i: + spks.add(bytes.fromhex(i['prevout']['scriptPubKey']['hex'])) + # gather output scripts + for o in tx['vout']: + if o['scriptPubKey']['type'] != 'nulldata': + spks.add(bytes.fromhex(o['scriptPubKey']['hex'])) + return spks diff --git a/test/functional/test_framework/siphash.py b/test/functional/test_framework/siphash.py index 85836845d4..5ad245cf1b 100644 --- a/test/functional/test_framework/siphash.py +++ b/test/functional/test_framework/siphash.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Specialized SipHash-2-4 implementations. +"""SipHash-2-4 implementation. -This implements SipHash-2-4 for 256-bit integers. +This implements SipHash-2-4. For convenience, an interface taking 256-bit +integers is provided in addition to the one accepting generic data. """ def rotl64(n, b): return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b + def siphash_round(v0, v1, v2, v3): v0 = (v0 + v1) & ((1 << 64) - 1) v1 = rotl64(v1, 13) @@ -27,37 +29,37 @@ def siphash_round(v0, v1, v2, v3): v2 = rotl64(v2, 32) return (v0, v1, v2, v3) -def siphash256(k0, k1, h): - n0 = h & ((1 << 64) - 1) - n1 = (h >> 64) & ((1 << 64) - 1) - n2 = (h >> 128) & ((1 << 64) - 1) - n3 = (h >> 192) & ((1 << 64) - 1) + +def siphash(k0, k1, data): + assert(type(data) == bytes) v0 = 0x736f6d6570736575 ^ k0 v1 = 0x646f72616e646f6d ^ k1 v2 = 0x6c7967656e657261 ^ k0 - v3 = 0x7465646279746573 ^ k1 ^ n0 - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0 ^= n0 - v3 ^= n1 - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0 ^= n1 - v3 ^= n2 + v3 = 0x7465646279746573 ^ k1 + c = 0 + t = 0 + for d in data: + t |= d << (8 * (c % 8)) + c = (c + 1) & 0xff + if (c & 7) == 0: + v3 ^= t + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= t + t = 0 + t = t | (c << 56) + v3 ^= t v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0 ^= n2 - v3 ^= n3 - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0 ^= n3 - v3 ^= 0x2000000000000000 - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) - v0 ^= 0x2000000000000000 - v2 ^= 0xFF + v0 ^= t + v2 ^= 0xff v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) return v0 ^ v1 ^ v2 ^ v3 + + +def siphash256(k0, k1, num): + assert(type(num) == int) + return siphash(k0, k1, num.to_bytes(32, 'little')) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index e20de8ea8e..e2c13a6705 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -136,6 +136,7 @@ BASE_SCRIPTS = [ # vv Tests less than 30s vv 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', + 'wallet_fast_rescan.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', 'rpc_invalid_address_message.py', diff --git a/test/functional/wallet_fast_rescan.py b/test/functional/wallet_fast_rescan.py new file mode 100755 index 0000000000..3b8ae8eb92 --- /dev/null +++ b/test/functional/wallet_fast_rescan.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that fast rescan using block filters for descriptor wallets detects + top-ups correctly and finds the same transactions than the slow variant.""" +import os +from typing import List + +from test_framework.descriptors import descsum_create +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import TestNode +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet +from test_framework.wallet_util import get_generate_key + + +KEYPOOL_SIZE = 100 # smaller than default size to speed-up test +NUM_DESCRIPTORS = 9 # number of descriptors (8 default ranged ones + 1 fixed non-ranged one) +NUM_BLOCKS = 6 # number of blocks to mine + + +class WalletFastRescanTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [[f'-keypool={KEYPOOL_SIZE}', '-blockfilterindex=1']] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + def get_wallet_txids(self, node: TestNode, wallet_name: str) -> List[str]: + w = node.get_wallet_rpc(wallet_name) + txs = w.listtransactions('*', 1000000) + return [tx['txid'] for tx in txs] + + def run_test(self): + node = self.nodes[0] + wallet = MiniWallet(node) + wallet.rescan_utxos() + + self.log.info("Create descriptor wallet with backup") + WALLET_BACKUP_FILENAME = os.path.join(node.datadir, 'wallet.bak') + node.createwallet(wallet_name='topup_test', descriptors=True) + w = node.get_wallet_rpc('topup_test') + fixed_key = get_generate_key() + print(w.importdescriptors([{"desc": descsum_create(f"wpkh({fixed_key.privkey})"), "timestamp": "now"}])) + descriptors = w.listdescriptors()['descriptors'] + assert_equal(len(descriptors), NUM_DESCRIPTORS) + w.backupwallet(WALLET_BACKUP_FILENAME) + + self.log.info(f"Create txs sending to end range address of each descriptor, triggering top-ups") + for i in range(NUM_BLOCKS): + self.log.info(f"Block {i+1}/{NUM_BLOCKS}") + for desc_info in w.listdescriptors()['descriptors']: + if 'range' in desc_info: + start_range, end_range = desc_info['range'] + addr = w.deriveaddresses(desc_info['desc'], [end_range, end_range])[0] + spk = bytes.fromhex(w.getaddressinfo(addr)['scriptPubKey']) + self.log.info(f"-> range [{start_range},{end_range}], last address {addr}") + else: + spk = bytes.fromhex(fixed_key.p2wpkh_script) + self.log.info(f"-> fixed non-range descriptor address {fixed_key.p2wpkh_addr}") + wallet.send_to(from_node=node, scriptPubKey=spk, amount=10000) + self.generate(node, 1) + + self.log.info("Import wallet backup with block filter index") + with node.assert_debug_log(['fast variant using block filters']): + node.restorewallet('rescan_fast', WALLET_BACKUP_FILENAME) + txids_fast = self.get_wallet_txids(node, 'rescan_fast') + + self.log.info("Import non-active descriptors with block filter index") + node.createwallet(wallet_name='rescan_fast_nonactive', descriptors=True, disable_private_keys=True, blank=True) + with node.assert_debug_log(['fast variant using block filters']): + w = node.get_wallet_rpc('rescan_fast_nonactive') + w.importdescriptors([{"desc": descriptor['desc'], "timestamp": 0} for descriptor in descriptors]) + txids_fast_nonactive = self.get_wallet_txids(node, 'rescan_fast_nonactive') + + self.restart_node(0, [f'-keypool={KEYPOOL_SIZE}', '-blockfilterindex=0']) + self.log.info("Import wallet backup w/o block filter index") + with node.assert_debug_log(['slow variant inspecting all blocks']): + node.restorewallet("rescan_slow", WALLET_BACKUP_FILENAME) + txids_slow = self.get_wallet_txids(node, 'rescan_slow') + + self.log.info("Import non-active descriptors w/o block filter index") + node.createwallet(wallet_name='rescan_slow_nonactive', descriptors=True, disable_private_keys=True, blank=True) + with node.assert_debug_log(['slow variant inspecting all blocks']): + w = node.get_wallet_rpc('rescan_slow_nonactive') + w.importdescriptors([{"desc": descriptor['desc'], "timestamp": 0} for descriptor in descriptors]) + txids_slow_nonactive = self.get_wallet_txids(node, 'rescan_slow_nonactive') + + self.log.info("Verify that all rescans found the same txs in slow and fast variants") + assert_equal(len(txids_slow), NUM_DESCRIPTORS * NUM_BLOCKS) + assert_equal(len(txids_fast), NUM_DESCRIPTORS * NUM_BLOCKS) + assert_equal(len(txids_slow_nonactive), NUM_DESCRIPTORS * NUM_BLOCKS) + assert_equal(len(txids_fast_nonactive), NUM_DESCRIPTORS * NUM_BLOCKS) + assert_equal(sorted(txids_slow), sorted(txids_fast)) + assert_equal(sorted(txids_slow_nonactive), sorted(txids_fast_nonactive)) + + +if __name__ == '__main__': + WalletFastRescanTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 5609ac9bf5..db3a8a2efa 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -98,7 +98,7 @@ class WalletSignerTest(BitcoinTestFramework): # ) # self.clear_mock_result(self.nodes[1]) - assert_equal(hww.getwalletinfo()["keypoolsize"], 30) + assert_equal(hww.getwalletinfo()["keypoolsize"], 40) address1 = hww.getnewaddress(address_type="bech32") assert_equal(address1, "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g") @@ -121,6 +121,13 @@ class WalletSignerTest(BitcoinTestFramework): assert_equal(address_info['ismine'], True) assert_equal(address_info['hdkeypath'], "m/44'/1'/0'/0/0") + address4 = hww.getnewaddress(address_type="bech32m") + assert_equal(address4, "bcrt1phw4cgpt6cd30kz9k4wkpwm872cdvhss29jga2xpmftelhqll62ms4e9sqj") + address_info = hww.getaddressinfo(address4) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/86'/1'/0'/0/0") + self.log.info('Test walletdisplayaddress') result = hww.walletdisplayaddress(address1) assert_equal(result, {"address": address1}) @@ -133,7 +140,7 @@ class WalletSignerTest(BitcoinTestFramework): self.clear_mock_result(self.nodes[1]) self.log.info('Prepare mock PSBT') - self.nodes[0].sendtoaddress(address1, 1) + self.nodes[0].sendtoaddress(address4, 1) self.generate(self.nodes[0], 1) # Load private key into wallet to generate a signed PSBT for the mock @@ -142,14 +149,14 @@ class WalletSignerTest(BitcoinTestFramework): assert mock_wallet.getwalletinfo()['private_keys_enabled'] result = mock_wallet.importdescriptors([{ - "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0", + "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#0jtt2jc9", "timestamp": 0, "range": [0,1], "internal": False, "active": True }, { - "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh", + "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#7xw2h8ga", "timestamp": 0, "range": [0, 0], "internal": True, |