diff options
49 files changed, 1046 insertions, 380 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index eb2d32e1bc..08132edb7d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -54,5 +54,5 @@ test_script: - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - ps: python test\util\bitcoin-util-test.py - cmd: python test\util\rpcauth-test.py -- cmd: python test\functional\test_runner.py --ci --force --quiet --combinedlogslen=4000 --failfast +- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast deploy: off diff --git a/.travis.yml b/.travis.yml index 873b33b05b..d5086b084a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,19 +66,21 @@ jobs: BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" - stage: test - name: 'Win32 [GOAL: deploy] [no gui tests]' + name: 'Win32 [GOAL: deploy] [no gui or functional tests]' env: >- HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32" + RUN_FUNCTIONAL_TESTS=false GOAL="deploy" BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" - stage: test - name: 'Win64 [GOAL: deploy] [no gui tests]' + name: 'Win64 [GOAL: deploy] [no gui or functional tests]' env: >- HOST=x86_64-w64-mingw32 PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" + RUN_FUNCTIONAL_TESTS=false GOAL="deploy" BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" diff --git a/doc/release-notes-14491.md b/doc/release-notes-14491.md new file mode 100644 index 0000000000..1cf36e85cf --- /dev/null +++ b/doc/release-notes-14491.md @@ -0,0 +1,5 @@ +Descriptor import support +--------------------- + +The `importmulti` RPC now supports importing of addresses from descriptors. A "desc" parameter can be provided instead of the "scriptPubKey" in a request, as well as an optional range for ranged descriptors to specify the start and end of the range to import. More information about +descriptors can be found [here](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). diff --git a/doc/release-notes-14667.md b/doc/release-notes-14667.md new file mode 100644 index 0000000000..5cb1d0aee7 --- /dev/null +++ b/doc/release-notes-14667.md @@ -0,0 +1,4 @@ +New RPC methods +------------ + +- `deriveaddresses` returns one or more addresses corresponding to an [output descriptor](/doc/descriptors.md). diff --git a/doc/release-notes-14941.md b/doc/release-notes-14941.md deleted file mode 100644 index c3820d0368..0000000000 --- a/doc/release-notes-14941.md +++ /dev/null @@ -1,5 +0,0 @@ -Miscellaneous RPC changes ------------- - -- The `unloadwallet` RPC is now synchronous, meaning that it blocks until the - wallet is fully unloaded. diff --git a/doc/release-notes-14982.md b/doc/release-notes-14982.md deleted file mode 100644 index 3f0bf8aacd..0000000000 --- a/doc/release-notes-14982.md +++ /dev/null @@ -1,5 +0,0 @@ -New RPCs --------- - -- The RPC `getrpcinfo` returns runtime details of the RPC server. At the moment - it returns the active commands and the corresponding execution time. diff --git a/doc/release-notes-15226.md b/doc/release-notes-15226.md new file mode 100644 index 0000000000..3be84db3e9 --- /dev/null +++ b/doc/release-notes-15226.md @@ -0,0 +1,8 @@ +Miscellaneous RPC changes +------------ + +- The RPC `createwallet` now has an optional `blank` argument that can be used to create a blank wallet. +Blank wallets do not have any keys or HD seed. +They cannot be opened in software older than 0.18. +Once a blank wallet has a HD seed set (by using `sethdseed`) or private keys, scripts, addresses, and other watch only things have been imported, the wallet is no longer blank and can be opened in 0.17.x. +Encrypting a blank wallet will also set a HD seed for it. diff --git a/doc/release-notes.md b/doc/release-notes.md index c23a7f6e0a..113b8c07d0 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -116,7 +116,8 @@ Configuration option changes defaults to being off, so that changes in policy and disconnect/ban behavior will not cause a node that is whitelisting another to be dropped by peers. Users can still explicitly enable this behavior with the command line option - (and may want to consider letting the Bitcoin Core project know about their + (and may want to consider [contacting](https://bitcoincore.org/en/contact/) + the Bitcoin Core project to let us know about their use-case, as this feature could be deprecated in the future). Documentation @@ -130,6 +131,10 @@ Documentation to the [REST interface documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md) indicating that the same rules apply. +- Further information is added to the [JSON-RPC + documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) + about how to secure this interface. + - A new [document](https://github.com/bitcoin/bitcoin/blob/master/doc/bitcoin-conf.md) about the `bitcoin.conf` file describes how to use it to configure Bitcoin Core. @@ -190,14 +195,18 @@ Deprecated or removed RPCs New RPCs -------- -- A new `getnodeaddresses` RPC returns peer addresses known to this +- The `getnodeaddresses` RPC returns peer addresses known to this node. It may be used to find nodes to connect to without using a DNS seeder. -- A new `listwalletdir` RPC returns a list of wallets in the wallet +- The `listwalletdir` RPC returns a list of wallets in the wallet directory (either the default wallet directory or the directory configured by the `-walletdir` parameter). +- The `getrpcinfo` returns runtime details of the RPC server. At the + moment, it returns an array of the currently active commands and how + long they've been running. + Updated RPCs ------------ @@ -260,6 +269,16 @@ in the Low-level Changes section below. 2. If no blockhash is provided, check the mempool. 3. If no blockhash is provided but txindex is enabled, also check txindex. +- The `unloadwallet` RPC is now synchronous, meaning it will not return + until the wallet is fully unloaded. + +REST changes +------------ + +- A new `/rest/blockhashbyheight/` endpoint is added for fetching the + hash of the block in the current best blockchain based on its height + (how many blocks it is after the Genesis Block). + Graphical User Interface (GUI) ------------------------------ @@ -282,6 +301,16 @@ Graphical User Interface (GUI) CFLAGS="-mmacosx-version-min=10.11" for setting the deployment sdk version) +Tools +---- + +- A new `bitcoin-wallet` tool is now distributed alongside Bitcoin + Core's other executables. Without needing to use any RPCs, this tool + can currently create a new wallet file or display some basic + information about an existing wallet, such as whether the wallet is + encrypted, whether it uses an HD seed, how many transactions it + contains, and how many address book entries it has. + Low-level changes ================= @@ -306,6 +335,32 @@ Configuration deterministic wallets. This release makes specifying `-usehd` an invalid configuration option. +Network +------- + +- This release allows peers that your node automatically disconnected + for misbehavior (e.g. sending invalid data) to reconnect to your node + if you have unused incoming connection slots. If your slots fill up, + a misbehaving node will be disconnected to make room for nodes without + a history of problems (unless the misbehaving node helps your node in + some other way, such as by connecting to a part of the Internet from + which you don't have many other peers). Previously, Bitcoin Core + banned the IP addresses of misbehaving peers for a period of time + (default of 1 day); this was easily circumvented by attackers with + multiple IP addresses. If you manually ban a peer, such as by using + the `setban` RPC, all connections from that peer will still be + rejected. + +Security +-------- + +- This release changes the Random Number Generator (RNG) used from + OpenSSL to Bitcoin Core's own implementation, although entropy + gathered by Bitcoin Core is fed out to OpenSSL and then read back in + when the program needs strong randomness. This moves Bitcoin Core a + little closer to no longer needing to depend on OpenSSL, a dependency + that has caused security issues in the past. + Changes for particular platforms -------------------------------- diff --git a/doc/translation_strings_policy.md b/doc/translation_strings_policy.md index 737d11f045..634aca3559 100644 --- a/doc/translation_strings_policy.md +++ b/doc/translation_strings_policy.md @@ -33,25 +33,25 @@ General recommendations Try not to burden translators with translating messages that are e.g. slight variations of other messages. In the GUI, avoid the use of text where an icon or symbol will do. -Make sure that placeholder texts in forms don't end up in the list of strings to be translated (use `<string notr="true">`). +Make sure that placeholder texts in forms do not end up in the list of strings to be translated (use `<string notr="true">`). ### Make translated strings understandable -Try to write translation strings in an understandable way, for both the user and the translator. Avoid overly technical or detailed messages +Try to write translation strings in an understandable way, for both the user and the translator. Avoid overly technical or detailed messages. ### Do not translate internal errors -Do not translate internal errors, or log messages, or messages that appear on the RPC interface. If an error is to be shown to the user, -use a translatable generic message, then log the detailed message to the log. E.g. "A fatal internal error occurred, see debug.log for details". +Do not translate internal errors, log messages, or messages that appear on the RPC interface. If an error is to be shown to the user, +use a translatable generic message, then log the detailed message to the log. E.g., "A fatal internal error occurred, see debug.log for details". This helps troubleshooting; if the error is the same for everyone, the likelihood is increased that it can be found using a search engine. ### Avoid fragments -Avoid dividing up a message into fragments. Translators see every string separately, so may misunderstand the context if the messages are not self-contained. +Avoid dividing up a message into fragments. Translators see every string separately, so they may misunderstand the context if the messages are not self-contained. ### Avoid HTML in translation strings -There have been difficulties with use of HTML in translation strings; translators should not be able to accidentally affect the formatting of messages. +There have been difficulties with the use of HTML in translation strings; translators should not be able to accidentally affect the formatting of messages. This may sometimes be at conflict with the recommendation in the previous section. ### Plurals @@ -66,7 +66,7 @@ Plurals can be complex in some languages. A quote from the gettext documentation 25-31 pliko'w and so on -In Qt code use tr's third argument for optional plurality. For example: +In Qt code, use tr's third argument for optional plurality. For example: tr("%n hour(s)","",secs/HOUR_IN_SECONDS); tr("%n day(s)","",secs/DAY_IN_SECONDS); @@ -82,7 +82,7 @@ This adds `<numerusform>`s to the respective `.ts` file, which can be translated </translation> </message> -Where it is possible try to avoid embedding numbers into the flow of the string at all. e.g. +Where possible, try to avoid embedding numbers into the flow of the string at all. E.g., WARNING: check your network connection, %d blocks received in the last %d hours (%d expected) diff --git a/src/init.cpp b/src/init.cpp index 09e28f4006..0013319ad5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1097,6 +1097,22 @@ bool AppInitParameterInteraction() dustRelayFee = CFeeRate(n); } + // This is required by both the wallet and node + if (gArgs.IsArgSet("-maxtxfee")) + { + CAmount nMaxFee = 0; + if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) + return InitError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); + if (nMaxFee > HIGH_MAX_TX_FEE) + InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); + maxTxFee = nMaxFee; + if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) + { + return InitError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + gArgs.GetArg("-maxtxfee", ""), ::minRelayTxFee.ToString())); + } + } + fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index a2cae2a7a7..03b47bd3b5 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -212,6 +212,7 @@ public: } std::vector<std::string> getDestValues(const std::string& prefix) override { + LOCK(m_wallet.cs_wallet); return m_wallet.GetDestValues(prefix); } void lockCoin(const COutPoint& output) override @@ -463,6 +464,7 @@ public: } unsigned int getConfirmTarget() override { return m_wallet.m_confirm_target; } bool hdEnabled() override { return m_wallet.IsHDEnabled(); } + bool canGetAddresses() override { return m_wallet.CanGetAddresses(); } bool IsWalletFlagSet(uint64_t flag) override { return m_wallet.IsWalletFlagSet(flag); } OutputType getDefaultAddressType() override { return m_wallet.m_default_address_type; } OutputType getDefaultChangeType() override { return m_wallet.m_default_change_type; } diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 72c64ded01..a86212356c 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -235,6 +235,9 @@ public: // Return whether HD enabled. virtual bool hdEnabled() = 0; + // Return whether the wallet is blank. + virtual bool canGetAddresses() = 0; + // check if a certain wallet flag is set. virtual bool IsWalletFlagSet(uint64_t flag) = 0; diff --git a/src/key_io.cpp b/src/key_io.cpp index d998089535..1d53a5e074 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -142,7 +142,9 @@ CKey DecodeSecret(const std::string& str) key.Set(data.begin() + privkey_prefix.size(), data.begin() + privkey_prefix.size() + 32, compressed); } } - memory_cleanse(data.data(), data.size()); + if (!data.empty()) { + memory_cleanse(data.data(), data.size()); + } return key; } diff --git a/src/net.cpp b/src/net.cpp index d52d5b7cf5..87f1ef0577 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -85,8 +85,6 @@ std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; std::string strSubVersion; -limitedmap<uint256, int64_t> mapAlreadyAskedFor(MAX_INV_SZ); - void CConnman::AddOneShot(const std::string& strDest) { LOCK(cs_vOneShots); @@ -2644,40 +2642,6 @@ CNode::~CNode() CloseSocket(hSocket); } -void CNode::AskFor(const CInv& inv) -{ - if (mapAskFor.size() > MAPASKFOR_MAX_SZ || setAskFor.size() > SETASKFOR_MAX_SZ) - return; - // a peer may not have multiple non-responded queue positions for a single inv item - if (!setAskFor.insert(inv.hash).second) - return; - - // We're using mapAskFor as a priority queue, - // the key is the earliest time the request can be sent - int64_t nRequestTime; - limitedmap<uint256, int64_t>::const_iterator it = mapAlreadyAskedFor.find(inv.hash); - if (it != mapAlreadyAskedFor.end()) - nRequestTime = it->second; - else - nRequestTime = 0; - LogPrint(BCLog::NET, "askfor %s %d (%s) peer=%d\n", inv.ToString(), nRequestTime, FormatISO8601Time(nRequestTime/1000000), id); - - // Make sure not to reuse time indexes to keep things in the same order - int64_t nNow = GetTimeMicros() - 1000000; - static int64_t nLastTime; - ++nLastTime; - nNow = std::max(nNow, nLastTime); - nLastTime = nNow; - - // Each retry is 2 minutes after the last - nRequestTime = std::max(nRequestTime + 2 * 60 * 1000000, nNow); - if (it != mapAlreadyAskedFor.end()) - mapAlreadyAskedFor.update(it, nRequestTime); - else - mapAlreadyAskedFor.insert(std::make_pair(inv.hash, nRequestTime)); - mapAskFor.insert(std::make_pair(nRequestTime, inv)); -} - bool CConnman::NodeFullyConnected(const CNode* pnode) { return pnode && pnode->fSuccessfullyConnected && !pnode->fDisconnect; @@ -67,10 +67,6 @@ static const bool DEFAULT_UPNP = USE_UPNP; #else static const bool DEFAULT_UPNP = false; #endif -/** The maximum number of entries in mapAskFor */ -static const size_t MAPASKFOR_MAX_SZ = MAX_INV_SZ; -/** The maximum number of entries in setAskFor (larger due to getdata latency)*/ -static const size_t SETASKFOR_MAX_SZ = 2 * MAX_INV_SZ; /** The maximum number of peer connections to maintain. */ static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 125; /** The default for -maxuploadtarget. 0 = Unlimited */ @@ -178,7 +174,18 @@ public: CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); - void Stop(); + + // TODO: Remove NO_THREAD_SAFETY_ANALYSIS. Lock cs_vNodes before reading the variable vNodes. + // + // When removing NO_THREAD_SAFETY_ANALYSIS be aware of the following lock order requirements: + // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraOutboundCount + // which locks cs_vNodes. + // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which + // locks cs_vNodes. + // + // Thus the implicit locking order requirement is: (1) cs_main, (2) g_cs_orphans, (3) cs_vNodes. + void Stop() NO_THREAD_SAFETY_ANALYSIS; + void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; @@ -386,7 +393,7 @@ private: CCriticalSection cs_vOneShots; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); CCriticalSection cs_vAddedNodes; - std::vector<CNode*> vNodes; + std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); std::list<CNode*> vNodesDisconnected; mutable CCriticalSection cs_vNodes; std::atomic<NodeId> nLastNodeId{0}; @@ -514,8 +521,6 @@ extern bool fDiscover; extern bool fListen; extern bool fRelayTxes; -extern limitedmap<uint256, int64_t> mapAlreadyAskedFor; - /** Subversion as sent to the P2P network in `version` messages */ extern std::string strSubVersion; @@ -704,8 +709,6 @@ public: // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); CCriticalSection cs_inventory; - std::set<uint256> setAskFor; - std::multimap<int64_t, CInv> mapAskFor; int64_t nNextInvSend{0}; // Used for headers announcements - unfiltered blocks to relay std::vector<uint256> vBlockHashesToAnnounce GUARDED_BY(cs_inventory); @@ -852,8 +855,6 @@ public: vBlockHashesToAnnounce.push_back(hash); } - void AskFor(const CInv& inv); - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 62b7d4e966..5927a14a6e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -64,6 +64,21 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; +/** Maximum number of in-flight transactions from a peer */ +static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; +/** Maximum number of announced transactions from a peer */ +static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 2 * MAX_INV_SZ; +/** How many microseconds to delay requesting transactions from inbound peers */ +static constexpr int64_t INBOUND_PEER_TX_DELAY = 2 * 1000000; +/** How long to wait (in microseconds) before downloading a transaction from an additional peer */ +static constexpr int64_t GETDATA_TX_INTERVAL = 60 * 1000000; +/** Maximum delay (in microseconds) for transaction requests to avoid biasing some peers over others. */ +static constexpr int64_t MAX_GETDATA_RANDOM_DELAY = 2 * 1000000; +static_assert(INBOUND_PEER_TX_DELAY >= MAX_GETDATA_RANDOM_DELAY, +"To preserve security, MAX_GETDATA_RANDOM_DELAY should not exceed INBOUND_PEER_DELAY"); +/** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ +static const unsigned int MAX_GETDATA_SZ = 1000; + struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -274,6 +289,66 @@ struct CNodeState { //! Time of last new block announcement int64_t m_last_block_announcement; + /* + * State associated with transaction download. + * + * Tx download algorithm: + * + * When inv comes in, queue up (process_time, txid) inside the peer's + * CNodeState (m_tx_process_time) as long as m_tx_announced for the peer + * isn't too big (MAX_PEER_TX_ANNOUNCEMENTS). + * + * The process_time for a transaction is set to nNow for outbound peers, + * nNow + 2 seconds for inbound peers. This is the time at which we'll + * consider trying to request the transaction from the peer in + * SendMessages(). The delay for inbound peers is to allow outbound peers + * a chance to announce before we request from inbound peers, to prevent + * an adversary from using inbound connections to blind us to a + * transaction (InvBlock). + * + * When we call SendMessages() for a given peer, + * we will loop over the transactions in m_tx_process_time, looking + * at the transactions whose process_time <= nNow. We'll request each + * such transaction that we don't have already and that hasn't been + * requested from another peer recently, up until we hit the + * MAX_PEER_TX_IN_FLIGHT limit for the peer. Then we'll update + * g_already_asked_for for each requested txid, storing the time of the + * GETDATA request. We use g_already_asked_for to coordinate transaction + * requests amongst our peers. + * + * For transactions that we still need but we have already recently + * requested from some other peer, we'll reinsert (process_time, txid) + * back into the peer's m_tx_process_time at the point in the future at + * which the most recent GETDATA request would time out (ie + * GETDATA_TX_INTERVAL + the request time stored in g_already_asked_for). + * We add an additional delay for inbound peers, again to prefer + * attempting download from outbound peers first. + * We also add an extra small random delay up to 2 seconds + * to avoid biasing some peers over others. (e.g., due to fixed ordering + * of peer processing in ThreadMessageHandler). + * + * When we receive a transaction from a peer, we remove the txid from the + * peer's m_tx_in_flight set and from their recently announced set + * (m_tx_announced). We also clear g_already_asked_for for that entry, so + * that if somehow the transaction is not accepted but also not added to + * the reject filter, then we will eventually redownload from other + * peers. + */ + struct TxDownloadState { + /* Track when to attempt download of announced transactions (process + * time in micros -> txid) + */ + std::multimap<int64_t, uint256> m_tx_process_time; + + //! Store all the transactions a peer has recently announced + std::set<uint256> m_tx_announced; + + //! Store transactions which were requested by us + std::set<uint256> m_tx_in_flight; + }; + + TxDownloadState m_tx_download; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -301,6 +376,9 @@ struct CNodeState { } }; +// Keeps track of the time (in microseconds) when transactions were requested last time +limitedmap<uint256, int64_t> g_already_asked_for GUARDED_BY(cs_main)(MAX_INV_SZ); + /** Map maintaining per-node state. */ static std::map<NodeId, CNodeState> mapNodeState GUARDED_BY(cs_main); @@ -591,6 +669,58 @@ static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vec } } +void EraseTxRequest(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + g_already_asked_for.erase(txid); +} + +int64_t GetTxRequestTime(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + auto it = g_already_asked_for.find(txid); + if (it != g_already_asked_for.end()) { + return it->second; + } + return 0; +} + +void UpdateTxRequestTime(const uint256& txid, int64_t request_time) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + auto it = g_already_asked_for.find(txid); + if (it == g_already_asked_for.end()) { + g_already_asked_for.insert(std::make_pair(txid, request_time)); + } else { + g_already_asked_for.update(it, request_time); + } +} + + +void RequestTx(CNodeState* state, const uint256& txid, int64_t nNow) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + CNodeState::TxDownloadState& peer_download_state = state->m_tx_download; + if (peer_download_state.m_tx_announced.size() >= MAX_PEER_TX_ANNOUNCEMENTS || peer_download_state.m_tx_announced.count(txid)) { + // Too many queued announcements from this peer, or we already have + // this announcement + return; + } + peer_download_state.m_tx_announced.insert(txid); + + int64_t process_time; + int64_t last_request_time = GetTxRequestTime(txid); + // First time requesting this tx + if (last_request_time == 0) { + process_time = nNow; + } else { + // Randomize the delay to avoid biasing some peers over others (such as due to + // fixed ordering of peer processing in ThreadMessageHandler) + process_time = last_request_time + GETDATA_TX_INTERVAL + GetRand(MAX_GETDATA_RANDOM_DELAY); + } + + // We delay processing announcements from non-preferred (eg inbound) peers + if (!state->fPreferredDownload) process_time += INBOUND_PEER_TX_DELAY; + + peer_download_state.m_tx_process_time.emplace(process_time, txid); +} + } // namespace // This function is used for testing the stale tip eviction logic, see @@ -1945,6 +2075,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); uint32_t nFetchFlags = GetFetchFlags(pfrom); + int64_t nNow = GetTimeMicros(); for (CInv &inv : vInv) { @@ -1976,7 +2107,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->GetId()); } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { - pfrom->AskFor(inv); + RequestTx(State(pfrom->GetId()), inv.hash, nNow); } } } @@ -2211,8 +2342,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fMissingInputs = false; CValidationState state; - pfrom->setAskFor.erase(inv.hash); - mapAlreadyAskedFor.erase(inv.hash); + CNodeState* nodestate = State(pfrom->GetId()); + nodestate->m_tx_download.m_tx_announced.erase(inv.hash); + nodestate->m_tx_download.m_tx_in_flight.erase(inv.hash); + EraseTxRequest(inv.hash); std::list<CTransactionRef> lRemovedTxn; @@ -2303,10 +2436,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (!fRejectedParents) { uint32_t nFetchFlags = GetFetchFlags(pfrom); + int64_t nNow = GetTimeMicros(); + for (const CTxIn& txin : tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); pfrom->AddInventoryKnown(_inv); - if (!AlreadyHave(_inv)) pfrom->AskFor(_inv); + if (!AlreadyHave(_inv)) RequestTx(State(pfrom->GetId()), _inv.hash, nNow); } AddOrphanTx(ptx, pfrom->GetId()); @@ -3731,24 +3866,39 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // // Message: getdata (non-blocks) // - while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) - { - const CInv& inv = (*pto->mapAskFor.begin()).second; - if (!AlreadyHave(inv)) - { - LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); - vGetData.push_back(inv); - if (vGetData.size() >= 1000) - { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - vGetData.clear(); + auto& tx_process_time = state.m_tx_download.m_tx_process_time; + while (!tx_process_time.empty() && tx_process_time.begin()->first <= nNow && state.m_tx_download.m_tx_in_flight.size() < MAX_PEER_TX_IN_FLIGHT) { + const uint256& txid = tx_process_time.begin()->second; + CInv inv(MSG_TX | GetFetchFlags(pto), txid); + if (!AlreadyHave(inv)) { + // If this transaction was last requested more than 1 minute ago, + // then request. + int64_t last_request_time = GetTxRequestTime(inv.hash); + if (last_request_time <= nNow - GETDATA_TX_INTERVAL) { + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); + vGetData.push_back(inv); + if (vGetData.size() >= MAX_GETDATA_SZ) { + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + vGetData.clear(); + } + UpdateTxRequestTime(inv.hash, nNow); + state.m_tx_download.m_tx_in_flight.insert(inv.hash); + } else { + // This transaction is in flight from someone else; queue + // up processing to happen after the download times out + // (with a slight delay for inbound peers, to prefer + // requests to outbound peers). + RequestTx(&state, txid, nNow); } } else { - //If we're not going to ask, don't expect a response. - pto->setAskFor.erase(inv.hash); + // We have already seen this transaction, no need to download. + state.m_tx_download.m_tx_announced.erase(inv.hash); + state.m_tx_download.m_tx_in_flight.erase(inv.hash); } - pto->mapAskFor.erase(pto->mapAskFor.begin()); + tx_process_time.erase(tx_process_time.begin()); } + + if (!vGetData.empty()) connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index a0c7f8e3c2..58e45c2c02 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -182,16 +182,16 @@ bool CNetAddr::IsTor() const bool CNetAddr::IsLocal() const { - // IPv4 loopback - if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) - return true; + // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) + if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) + return true; - // IPv6 loopback (::1/128) - static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (memcmp(ip, pchLocal, 16) == 0) - return true; + // IPv6 loopback (::1/128) + static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; + if (memcmp(ip, pchLocal, 16) == 0) + return true; - return false; + return false; } bool CNetAddr::IsValid() const diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 3e414df1f0..7f5e92ea9f 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -95,6 +95,7 @@ void TestAddAddressesToSendBook() } auto check_addbook_size = [&wallet](int expected_size) { + LOCK(wallet->cs_wallet); QCOMPARE(static_cast<int>(wallet->mapAddressBook.size()), expected_size); }; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 2a9144bec9..f4f3be8f43 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -580,12 +580,7 @@ bool WalletModel::privateKeysDisabled() const bool WalletModel::canGetAddresses() const { - // The wallet can provide a fresh address if: - // * hdEnabled(): an HD seed is present; or - // * it is a legacy wallet, because: - // * !hdEnabled(): an HD seed is not present; and - // * !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS): private keys have not been disabled (which results in hdEnabled() == true) - return m_wallet->hdEnabled() || (!m_wallet->hdEnabled() && !m_wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + return m_wallet->canGetAddresses(); } QString WalletModel::getWalletName() const diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f95e22574c..c5694e6d55 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -68,6 +68,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 4, "subtractfeefrom" }, { "sendmany", 5 , "replaceable" }, { "sendmany", 6 , "conf_target" }, + { "deriveaddresses", 1, "begin" }, + { "deriveaddresses", 2, "end" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, @@ -159,6 +161,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "rescanblockchain", 0, "start_height"}, { "rescanblockchain", 1, "stop_height"}, { "createwallet", 1, "disable_private_keys"}, + { "createwallet", 2, "blank"}, { "getnodeaddresses", 0, "count"}, { "stop", 0, "wait" }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 200dfa107b..35f55b0141 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -31,16 +31,6 @@ #include <memory> #include <stdint.h> -unsigned int ParseConfirmTarget(const UniValue& value) -{ - int target = value.get_int(); - unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - if (target < 1 || (unsigned int)target > max_target) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid conf_target, must be between %u - %u", 1, max_target)); - } - return (unsigned int)target; -} - /** * Return average network hashes per second based on the last 'lookup' blocks, * or from the last difficulty change if 'lookup' is nonpositive. diff --git a/src/rpc/mining.h b/src/rpc/mining.h index 8d46273159..be9a973315 100644 --- a/src/rpc/mining.h +++ b/src/rpc/mining.h @@ -12,7 +12,4 @@ /** Generate blocks (mine) */ UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript); -/** Check bounds on a command line confirm target */ -unsigned int ParseConfirmTarget(const UniValue& value); - #endif diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 8850cf066b..9702dc47e8 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -16,6 +16,7 @@ #include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/descriptor.h> #include <timedata.h> #include <util/system.h> #include <util/strencodings.h> @@ -142,6 +143,95 @@ static UniValue createmultisig(const JSONRPCRequest& request) return result; } +UniValue deriveaddresses(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.empty() || request.params.size() > 3) { + throw std::runtime_error( + RPCHelpMan{"deriveaddresses", + {"\nDerives one or more addresses corresponding to an output descriptor.\n" + "Examples of output descriptors are:\n" + " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" + " wpkh(<pubkey>) Native segwit P2PKH outputs for the given pubkey\n" + " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" + " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" + "\nIn the above, <pubkey> either refers to a fixed public key in hexadecimal notation, or to an xpub/xprv optionally followed by one\n" + "or more path elements separated by \"/\", where \"h\" represents a hardened child key.\n" + "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, + { + {"descriptor", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The descriptor."}, + {"begin", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "", "If a ranged descriptor is used, this specifies the beginning of the range to import."}, + {"end", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "", "If a ranged descriptor is used, this specifies the end of the range to import."} + }, + RPCResult{ + "[ address ] (array) the derived addresses\n" + }, + RPCExamples{ + "First three native segwit receive addresses\n" + + HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)\" 0 2") + }}.ToString() + ); + } + + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VNUM, UniValue::VNUM}); + const std::string desc_str = request.params[0].get_str(); + + int range_begin = 0; + int range_end = 0; + + if (request.params.size() >= 2) { + if (request.params.size() == 2) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing range end parameter"); + } + range_begin = request.params[1].get_int(); + range_end = request.params[2].get_int(); + if (range_begin < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should be greater or equal than 0"); + } + if (range_begin > range_end) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range end should be equal to or greater than begin"); + } + } + + FlatSigningProvider provider; + auto desc = Parse(desc_str, provider); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor")); + } + + if (!desc->IsRange() && request.params.size() > 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } + + if (desc->IsRange() && request.params.size() == 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range must be specified for a ranged descriptor"); + } + + UniValue addresses(UniValue::VARR); + + for (int i = range_begin; i <= range_end; ++i) { + std::vector<CScript> scripts; + if (!desc->Expand(i, provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + for (const CScript &script : scripts) { + CTxDestination dest; + if (!ExtractDestination(script, dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Descriptor does not have a corresponding address")); + } + + addresses.push_back(EncodeDestination(dest)); + } + } + + // This should not be possible, but an assert seems overkill: + if (addresses.empty()) { + throw JSONRPCError(RPC_MISC_ERROR, "Unexpected empty result"); + } + + return addresses; +} + static UniValue verifymessage(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 3) @@ -473,6 +563,7 @@ static const CRPCCommand commands[] = { "control", "logging", &logging, {"include", "exclude"}}, { "util", "validateaddress", &validateaddress, {"address"} }, { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, + { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "begin", "end"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index b3b9d8af09..cce62bacef 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1518,6 +1518,9 @@ UniValue combinepsbt(const JSONRPCRequest& request) // Unserialize the transactions std::vector<PartiallySignedTransaction> psbtxs; UniValue txs = request.params[0].get_array(); + if (txs.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txs' cannot be empty"); + } for (unsigned int i = 0; i < txs.size(); ++i) { PartiallySignedTransaction psbtx; std::string error; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 4275cc09a8..aa5076cd8e 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -4,10 +4,12 @@ #include <key_io.h> #include <keystore.h> +#include <policy/fees.h> #include <rpc/protocol.h> #include <rpc/util.h> #include <tinyformat.h> #include <util/strencodings.h> +#include <validation.h> InitInterfaces* g_rpc_interfaces = nullptr; @@ -129,6 +131,16 @@ UniValue DescribeAddress(const CTxDestination& dest) return boost::apply_visitor(DescribeAddressVisitor(), dest); } +unsigned int ParseConfirmTarget(const UniValue& value) +{ + int target = value.get_int(); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + if (target < 1 || (unsigned int)target > max_target) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid conf_target, must be between %u - %u", 1, max_target)); + } + return (unsigned int)target; +} + struct Section { Section(const std::string& left, const std::string& right) : m_left{left}, m_right{right} {} diff --git a/src/rpc/util.h b/src/rpc/util.h index 4a9d4be787..d34c9cfdbb 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -28,6 +28,9 @@ CScript CreateMultisigRedeemscript(const int required, const std::vector<CPubKey UniValue DescribeAddress(const CTxDestination& dest); +//! Parse a confirm target option and raise an RPC error if it is invalid. +unsigned int ParseConfirmTarget(const UniValue& value); + struct RPCArg { enum class Type { OBJ, diff --git a/src/scheduler.h b/src/scheduler.h index 6d7f42cf9f..436f661c59 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -45,13 +45,13 @@ public: // Call func at/after time t void schedule(Function f, boost::chrono::system_clock::time_point t=boost::chrono::system_clock::now()); - // Convenience method: call f once deltaSeconds from now + // Convenience method: call f once deltaMilliSeconds from now void scheduleFromNow(Function f, int64_t deltaMilliSeconds); // Another convenience method: call f approximately - // every deltaSeconds forever, starting deltaSeconds from now. + // every deltaMilliSeconds forever, starting deltaMilliSeconds from now. // To be more precise: every time f is finished, it - // is rescheduled to run deltaSeconds later. If you + // is rescheduled to run deltaMilliSeconds later. If you // need more accurate scheduling, don't use this method. void scheduleEvery(Function f, int64_t deltaMilliSeconds); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 71b6ec7425..860f64bb11 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -169,11 +169,6 @@ BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); } -BOOST_AUTO_TEST_CASE(util_FormatISO8601Time) -{ - BOOST_CHECK_EQUAL(FormatISO8601Time(1317425777), "23:36:17Z"); -} - struct TestArgsManager : public ArgsManager { TestArgsManager() { m_network_only_args.clear(); } diff --git a/src/util/time.cpp b/src/util/time.cpp index 83a7937d8f..c0ede98701 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -97,14 +97,3 @@ std::string FormatISO8601Date(int64_t nTime) { #endif return strprintf("%04i-%02i-%02i", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday); } - -std::string FormatISO8601Time(int64_t nTime) { - struct tm ts; - time_t time_val = nTime; -#ifdef _MSC_VER - gmtime_s(&ts, &time_val); -#else - gmtime_r(&time_val, &ts); -#endif - return strprintf("%02i:%02i:%02iZ", ts.tm_hour, ts.tm_min, ts.tm_sec); -} diff --git a/src/util/time.h b/src/util/time.h index f2e2747434..68de1c156e 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -33,6 +33,5 @@ void MilliSleep(int64_t n); */ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); -std::string FormatISO8601Time(int64_t nTime); #endif // BITCOIN_UTIL_TIME_H diff --git a/src/validation.cpp b/src/validation.cpp index de9c0d96db..dbdc1afb35 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -152,7 +152,7 @@ private: public: CChain chainActive; - BlockMap mapBlockIndex; + BlockMap mapBlockIndex GUARDED_BY(cs_main); std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; CBlockIndex *pindexBestInvalid = nullptr; diff --git a/src/validation.h b/src/validation.h index b16d8438d7..49f73e4c9b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -151,7 +151,7 @@ extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; extern std::atomic_bool g_is_mempool_loaded; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; -extern BlockMap& mapBlockIndex; +extern BlockMap& mapBlockIndex GUARDED_BY(cs_main); extern uint64_t nLastBlockTx; extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; @@ -288,7 +288,7 @@ uint64_t CalculateCurrentUsage(); /** * Mark one block file as pruned. */ -void PruneOneBlockFile(const int fileNumber); +void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Actually unlink the specified files diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 1dc78255f6..a255177e36 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -182,7 +182,7 @@ bool CCryptoKeyStore::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no if (!SetCrypted()) return false; - bool keyPass = false; + bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys bool keyFail = false; CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); for (; mi != mapCryptedKeys.end(); ++mi) diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 87cd264c3d..20d540c8db 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -127,21 +127,6 @@ bool WalletInit::ParameterInteraction() const InitWarning(AmountHighWarn("-minrelaytxfee") + " " + _("The wallet will avoid paying less than the minimum relay fee.")); - if (gArgs.IsArgSet("-maxtxfee")) - { - CAmount nMaxFee = 0; - if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) - return InitError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); - if (nMaxFee > HIGH_MAX_TX_FEE) - InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); - maxTxFee = nMaxFee; - if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) - { - return InitError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), - gArgs.GetArg("-maxtxfee", ""), ::minRelayTxFee.ToString())); - } - } - return true; } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 32c36ceaeb..7552722a8e 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -9,6 +9,7 @@ #include <merkleblock.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/descriptor.h> #include <script/script.h> #include <script/standard.h> #include <sync.h> @@ -66,7 +67,7 @@ static std::string DecodeDumpString(const std::string &str) { return ret.str(); } -static bool GetWalletAddressesForKey(CWallet * const pwallet, const CKeyID &keyid, std::string &strAddr, std::string &strLabel) +static bool GetWalletAddressesForKey(CWallet* const pwallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { bool fLabelFound = false; CKey key; @@ -964,159 +965,273 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } } -static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data) { UniValue warnings(UniValue::VARR); - UniValue result(UniValue::VOBJ); - try { - // First ensure scriptPubKey has either a script or JSON with "address" string - const UniValue& scriptPubKey = data["scriptPubKey"]; - bool isScript = scriptPubKey.getType() == UniValue::VSTR; - if (!isScript && !(scriptPubKey.getType() == UniValue::VOBJ && scriptPubKey.exists("address"))) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "scriptPubKey must be string with script or JSON with address string"); + // First ensure scriptPubKey has either a script or JSON with "address" string + const UniValue& scriptPubKey = data["scriptPubKey"]; + bool isScript = scriptPubKey.getType() == UniValue::VSTR; + if (!isScript && !(scriptPubKey.getType() == UniValue::VOBJ && scriptPubKey.exists("address"))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "scriptPubKey must be string with script or JSON with address string"); + } + const std::string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); + + // Optional fields. + const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; + const std::string& witness_script_hex = data.exists("witnessscript") ? data["witnessscript"].get_str() : ""; + const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); + const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const bool watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + + if (data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for a non-descriptor import"); + } + + // Generate the script and destination for the scriptPubKey provided + CScript script; + if (!isScript) { + CTxDestination dest = DecodeDestination(output); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address \"" + output + "\""); + } + script = GetScriptForDestination(dest); + } else { + if (!IsHex(output)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid scriptPubKey \"" + output + "\""); + } + std::vector<unsigned char> vData(ParseHex(output)); + script = CScript(vData.begin(), vData.end()); + CTxDestination dest; + if (!ExtractDestination(script, dest) && !internal) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal must be set to true for nonstandard scriptPubKey imports."); } - const std::string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); + } + script_pub_keys.emplace(script); - // Optional fields. - const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; - const std::string& witness_script_hex = data.exists("witnessscript") ? data["witnessscript"].get_str() : ""; - const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); - const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); - const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; - const bool watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; - const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + // Parse all arguments + if (strRedeemScript.size()) { + if (!IsHex(strRedeemScript)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid redeem script \"" + strRedeemScript + "\": must be hex string"); + } + auto parsed_redeemscript = ParseHex(strRedeemScript); + import_data.redeemscript = MakeUnique<CScript>(parsed_redeemscript.begin(), parsed_redeemscript.end()); + } + if (witness_script_hex.size()) { + if (!IsHex(witness_script_hex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid witness script \"" + witness_script_hex + "\": must be hex string"); + } + auto parsed_witnessscript = ParseHex(witness_script_hex); + import_data.witnessscript = MakeUnique<CScript>(parsed_witnessscript.begin(), parsed_witnessscript.end()); + } + for (size_t i = 0; i < pubKeys.size(); ++i) { + const auto& str = pubKeys[i].get_str(); + if (!IsHex(str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" must be a hex string"); + } + auto parsed_pubkey = ParseHex(str); + CPubKey pubkey(parsed_pubkey.begin(), parsed_pubkey.end()); + if (!pubkey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" is not a valid public key"); + } + pubkey_map.emplace(pubkey.GetID(), pubkey); + } + for (size_t i = 0; i < keys.size(); ++i) { + const auto& str = keys[i].get_str(); + CKey key = DecodeSecret(str); + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + } + CPubKey pubkey = key.GetPubKey(); + CKeyID id = pubkey.GetID(); + if (pubkey_map.count(id)) { + pubkey_map.erase(id); + } + privkey_map.emplace(id, key); + } - // If private keys are disabled, abort if private keys are being imported - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !keys.isNull()) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); + + // Verify and process input data + have_solving_data = import_data.redeemscript || import_data.witnessscript || pubkey_map.size() || privkey_map.size(); + if (have_solving_data) { + // Match up data in import_data with the scriptPubKey in script. + auto error = RecurseImportData(script, import_data, ScriptContext::TOP); + + // Verify whether the watchonly option corresponds to the availability of private keys. + bool spendable = std::all_of(import_data.used_keys.begin(), import_data.used_keys.end(), [&](const std::pair<CKeyID, bool>& used_key){ return privkey_map.count(used_key.first) > 0; }); + if (!watchOnly && !spendable) { + warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); + } + if (watchOnly && spendable) { + warnings.push_back("All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."); } - // Generate the script and destination for the scriptPubKey provided - CScript script; - CTxDestination dest; - if (!isScript) { - dest = DecodeDestination(output); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address \"" + output + "\""); + // Check that all required keys for solvability are provided. + if (error.empty()) { + for (const auto& require_key : import_data.used_keys) { + if (!require_key.second) continue; // Not a required key + if (pubkey_map.count(require_key.first) == 0 && privkey_map.count(require_key.first) == 0) { + error = "some required keys are missing"; + } } - script = GetScriptForDestination(dest); + } + + if (!error.empty()) { + warnings.push_back("Importing as non-solvable: " + error + ". If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript."); + import_data = ImportData(); + pubkey_map.clear(); + privkey_map.clear(); + have_solving_data = false; } else { - if (!IsHex(output)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid scriptPubKey \"" + output + "\""); + // RecurseImportData() removes any relevant redeemscript/witnessscript from import_data, so we can use that to discover if a superfluous one was provided. + if (import_data.redeemscript) warnings.push_back("Ignoring redeemscript as this is not a P2SH script."); + if (import_data.witnessscript) warnings.push_back("Ignoring witnessscript as this is not a (P2SH-)P2WSH script."); + for (auto it = privkey_map.begin(); it != privkey_map.end(); ) { + auto oldit = it++; + if (import_data.used_keys.count(oldit->first) == 0) { + warnings.push_back("Ignoring irrelevant private key."); + privkey_map.erase(oldit); + } } - std::vector<unsigned char> vData(ParseHex(output)); - script = CScript(vData.begin(), vData.end()); - if (!ExtractDestination(script, dest) && !internal) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal must be set to true for nonstandard scriptPubKey imports."); + for (auto it = pubkey_map.begin(); it != pubkey_map.end(); ) { + auto oldit = it++; + auto key_data_it = import_data.used_keys.find(oldit->first); + if (key_data_it == import_data.used_keys.end() || !key_data_it->second) { + warnings.push_back("Ignoring public key \"" + HexStr(oldit->first) + "\" as it doesn't appear inside P2PKH or P2WPKH."); + pubkey_map.erase(oldit); + } } } + } - // Parse all arguments - ImportData import_data; - if (strRedeemScript.size()) { - if (!IsHex(strRedeemScript)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid redeem script \"" + strRedeemScript + "\": must be hex string"); - } - auto parsed_redeemscript = ParseHex(strRedeemScript); - import_data.redeemscript = MakeUnique<CScript>(parsed_redeemscript.begin(), parsed_redeemscript.end()); + return warnings; +} + +static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data) +{ + UniValue warnings(UniValue::VARR); + + const std::string& descriptor = data["desc"].get_str(); + FlatSigningProvider keys; + auto parsed_desc = Parse(descriptor, keys); + if (!parsed_desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Descriptor is invalid"); + } + + have_solving_data = parsed_desc->IsSolvable(); + const bool watch_only = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + + int64_t range_start = 0, range_end = 0; + if (!parsed_desc->IsRange() && data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } else if (parsed_desc->IsRange()) { + if (!data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor is ranged, please specify the range"); } - if (witness_script_hex.size()) { - if (!IsHex(witness_script_hex)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid witness script \"" + witness_script_hex + "\": must be hex string"); - } - auto parsed_witnessscript = ParseHex(witness_script_hex); - import_data.witnessscript = MakeUnique<CScript>(parsed_witnessscript.begin(), parsed_witnessscript.end()); + const UniValue& range = data["range"]; + range_start = range.exists("start") ? range["start"].get_int64() : 0; + if (!range.exists("end")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "End of range for descriptor must be specified"); } - std::map<CKeyID, CPubKey> pubkey_map; - for (size_t i = 0; i < pubKeys.size(); ++i) { - const auto& str = pubKeys[i].get_str(); - if (!IsHex(str)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" must be a hex string"); - } - auto parsed_pubkey = ParseHex(str); - CPubKey pubkey(parsed_pubkey.begin(), parsed_pubkey.end()); - if (!pubkey.IsFullyValid()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" is not a valid public key"); - } - pubkey_map.emplace(pubkey.GetID(), pubkey); + range_end = range["end"].get_int64(); + if (range_end < range_start || range_start < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid descriptor range specified"); } - std::map<CKeyID, CKey> privkey_map; - for (size_t i = 0; i < keys.size(); ++i) { - const auto& str = keys[i].get_str(); - CKey key = DecodeSecret(str); - if (!key.IsValid()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); - } - CPubKey pubkey = key.GetPubKey(); - CKeyID id = pubkey.GetID(); - if (pubkey_map.count(id)) { - pubkey_map.erase(id); - } + } + + const UniValue& priv_keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); + + FlatSigningProvider out_keys; + + // Expand all descriptors to get public keys and scripts. + // TODO: get private keys from descriptors too + for (int i = range_start; i <= range_end; ++i) { + std::vector<CScript> scripts_temp; + parsed_desc->Expand(i, keys, scripts_temp, out_keys); + std::copy(scripts_temp.begin(), scripts_temp.end(), std::inserter(script_pub_keys, script_pub_keys.end())); + } + + for (const auto& x : out_keys.scripts) { + import_data.import_scripts.emplace(x.second); + } + + std::copy(out_keys.pubkeys.begin(), out_keys.pubkeys.end(), std::inserter(pubkey_map, pubkey_map.end())); + + for (size_t i = 0; i < priv_keys.size(); ++i) { + const auto& str = priv_keys[i].get_str(); + CKey key = DecodeSecret(str); + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + } + CPubKey pubkey = key.GetPubKey(); + CKeyID id = pubkey.GetID(); + + // Check if this private key corresponds to a public key from the descriptor + if (!pubkey_map.count(id)) { + warnings.push_back("Ignoring irrelevant private key."); + } else { privkey_map.emplace(id, key); } + } + + // Check if all the public keys have corresponding private keys in the import for spendability. + // This does not take into account threshold multisigs which could be spendable without all keys. + // Thus, threshold multisigs without all keys will be considered not spendable here, even if they are, + // perhaps triggering a false warning message. This is consistent with the current wallet IsMine check. + bool spendable = std::all_of(pubkey_map.begin(), pubkey_map.end(), + [&](const std::pair<CKeyID, CPubKey>& used_key) { + return privkey_map.count(used_key.first) > 0; + }); + if (!watch_only && !spendable) { + warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); + } + if (watch_only && spendable) { + warnings.push_back("All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."); + } + + return warnings; +} + +static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + UniValue warnings(UniValue::VARR); + UniValue result(UniValue::VOBJ); + try { + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; // Internal addresses should not have a label if (internal && data.exists("label")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal addresses should not have a label"); } + const std::string& label = data.exists("label") ? data["label"].get_str() : ""; - // Verify and process input data - bool have_solving_data = import_data.redeemscript || import_data.witnessscript || pubkey_map.size() || privkey_map.size(); - if (have_solving_data) { - // Match up data in import_data with the scriptPubKey in script. - auto error = RecurseImportData(script, import_data, ScriptContext::TOP); - - // Verify whether the watchonly option corresponds to the availability of private keys. - bool spendable = std::all_of(import_data.used_keys.begin(), import_data.used_keys.end(), [&](const std::pair<CKeyID, bool>& used_key){ return privkey_map.count(used_key.first) > 0; }); - if (!watchOnly && !spendable) { - warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); - } - if (watchOnly && spendable) { - warnings.push_back("All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."); - } - - // Check that all required keys for solvability are provided. - if (error.empty()) { - for (const auto& require_key : import_data.used_keys) { - if (!require_key.second) continue; // Not a required key - if (pubkey_map.count(require_key.first) == 0 && privkey_map.count(require_key.first) == 0) { - error = "some required keys are missing"; - } - } - } + ImportData import_data; + std::map<CKeyID, CPubKey> pubkey_map; + std::map<CKeyID, CKey> privkey_map; + std::set<CScript> script_pub_keys; + bool have_solving_data; + + if (data.exists("scriptPubKey") && data.exists("desc")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Both a descriptor and a scriptPubKey should not be provided."); + } else if (data.exists("scriptPubKey")) { + warnings = ProcessImportLegacy(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data); + } else if (data.exists("desc")) { + warnings = ProcessImportDescriptor(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data); + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Either a descriptor or scriptPubKey must be provided."); + } - if (!error.empty()) { - warnings.push_back("Importing as non-solvable: " + error + ". If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript."); - import_data = ImportData(); - pubkey_map.clear(); - privkey_map.clear(); - have_solving_data = false; - } else { - // RecurseImportData() removes any relevant redeemscript/witnessscript from import_data, so we can use that to discover if a superfluous one was provided. - if (import_data.redeemscript) warnings.push_back("Ignoring redeemscript as this is not a P2SH script."); - if (import_data.witnessscript) warnings.push_back("Ignoring witnessscript as this is not a (P2SH-)P2WSH script."); - for (auto it = privkey_map.begin(); it != privkey_map.end(); ) { - auto oldit = it++; - if (import_data.used_keys.count(oldit->first) == 0) { - warnings.push_back("Ignoring irrelevant private key."); - privkey_map.erase(oldit); - } - } - for (auto it = pubkey_map.begin(); it != pubkey_map.end(); ) { - auto oldit = it++; - auto key_data_it = import_data.used_keys.find(oldit->first); - if (key_data_it == import_data.used_keys.end() || !key_data_it->second) { - warnings.push_back("Ignoring public key \"" + HexStr(oldit->first) + "\" as it doesn't appear inside P2PKH or P2WPKH."); - pubkey_map.erase(oldit); - } - } - } + // If private keys are disabled, abort if private keys are being imported + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !privkey_map.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } // Check whether we have any work to do - if (::IsMine(*pwallet, script) & ISMINE_SPENDABLE) { - throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + for (const CScript& script : script_pub_keys) { + if (::IsMine(*pwallet, script) & ISMINE_SPENDABLE) { + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script.begin(), script.end()) + "\")"); + } } // All good, time to import @@ -1146,14 +1261,18 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } } - if (!have_solving_data || !::IsMine(*pwallet, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated - if (!pwallet->AddWatchOnly(script, timestamp)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + + for (const CScript& script : script_pub_keys) { + if (!have_solving_data || !::IsMine(*pwallet, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated + if (!pwallet->AddWatchOnly(script, timestamp)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + } + CTxDestination dest; + ExtractDestination(script, dest); + if (!internal && IsValidDestination(dest)) { + pwallet->SetAddressBook(dest, label, "receive"); } - } - if (!internal) { - assert(IsValidDestination(dest)); - pwallet->SetAddressBook(dest, label, "receive"); } result.pushKV("success", UniValue(true)); @@ -1204,7 +1323,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) { {"", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "", { - {"scriptPubKey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "Type of scriptPubKey (string for script, json for address)", + {"desc", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "Descriptor to import. If using descriptor, do not also provide address/scriptPubKey, scripts, or pubkeys"}, + {"scriptPubKey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "Type of scriptPubKey (string for script, json for address). Should not be provided if using a descriptor", /* oneline_description */ "", {"\"<script>\" | { \"address\":\"<address>\" }", "string / json"} }, {"timestamp", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "Creation time of the key in seconds since epoch (Jan 1 1970 GMT),\n" @@ -1227,6 +1347,12 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) {"key", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", ""}, } }, + {"range", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "If a ranged descriptor is used, this specifies the start and end of the range to import", + { + {"start", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Start of the range to import"}, + {"end", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "End of the range to import (inclusive)"}, + } + }, {"internal", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Stating whether matching outputs should be treated as not incoming payments (also known as change)"}, {"watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Stating whether matching outputs should be considered watchonly."}, {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "''", "Label to assign to the address, only allowed with internal=false"}, diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e38ad3a0ba..9bbbdc6132 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -170,12 +170,18 @@ static UniValue getnewaddress(const JSONRPCRequest& request) }, }.ToString()); + // Belt and suspenders check for disabled private keys if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); } LOCK(pwallet->cs_wallet); + if (!pwallet->CanGetAddresses()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); + } + + // Parse the label first so we don't generate a key if there's an error std::string label; if (!request.params[0].isNull()) @@ -231,12 +237,17 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) }, }.ToString()); + // Belt and suspenders check for disabled private keys if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); } LOCK(pwallet->cs_wallet); + if (!pwallet->CanGetAddresses(true)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); + } + if (!pwallet->IsLocked()) { pwallet->TopUpKeyPool(); } @@ -1309,7 +1320,7 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) +static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { CAmount nFee; std::list<COutputEntry> listReceived; @@ -2578,13 +2589,14 @@ static UniValue loadwallet(const JSONRPCRequest& request) static UniValue createwallet(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) { throw std::runtime_error( RPCHelpMan{"createwallet", "\nCreates and loads a new wallet.\n", { {"wallet_name", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The name for the new wallet. If this is a path, the wallet will be created at the path location."}, {"disable_private_keys", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Disable the possibility of private keys (only watchonlys are possible in this mode)."}, + {"blank", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, }, RPCResult{ "{\n" @@ -2601,9 +2613,13 @@ static UniValue createwallet(const JSONRPCRequest& request) std::string error; std::string warning; - bool disable_privatekeys = false; - if (!request.params[1].isNull()) { - disable_privatekeys = request.params[1].get_bool(); + uint64_t flags = 0; + if (!request.params[1].isNull() && request.params[1].get_bool()) { + flags |= WALLET_FLAG_DISABLE_PRIVATE_KEYS; + } + + if (!request.params[2].isNull() && request.params[2].get_bool()) { + flags |= WALLET_FLAG_BLANK_WALLET; } WalletLocation location(request.params[0].get_str()); @@ -2616,7 +2632,7 @@ static UniValue createwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Wallet file verification failed: " + error); } - std::shared_ptr<CWallet> const wallet = CWallet::CreateWalletFromFile(*g_rpc_interfaces->chain, location, (disable_privatekeys ? (uint64_t)WALLET_FLAG_DISABLE_PRIVATE_KEYS : 0)); + std::shared_ptr<CWallet> const wallet = CWallet::CreateWalletFromFile(*g_rpc_interfaces->chain, location, flags); if (!wallet) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet creation failed."); } @@ -3886,7 +3902,7 @@ UniValue sethdseed(const JSONRPCRequest& request) LOCK(pwallet->cs_wallet); // Do not do anything to non-HD wallets - if (!pwallet->IsHDEnabled()) { + if (!pwallet->CanSupportFeature(FEATURE_HD)) { throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Start with -upgradewallet in order to upgrade a non-HD wallet to HD"); } @@ -4190,7 +4206,7 @@ static const CRPCCommand commands[] = { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys"} }, + { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index fcb34c3706..e674b2faea 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -44,6 +44,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = chainActive.Tip(); + LockAnnotation lock(::cs_main); auto locked_chain = chain->lock(); // Verify ScanForWalletTransactions accommodates a null start block. @@ -123,6 +124,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = chainActive.Tip(); + LockAnnotation lock(::cs_main); auto locked_chain = chain->lock(); // Prune the older block file. @@ -268,6 +270,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { + LockAnnotation lock(::cs_main); auto locked_chain = wallet.chain().lock(); auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); @@ -449,6 +452,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { auto chain = interfaces::MakeChain(); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); CPubKey pubkey; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index bdddcd718b..d38c15220f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -168,6 +168,7 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) { assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + assert(!IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); AssertLockHeld(cs_wallet); // mapKeyMetadata bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets @@ -177,7 +178,7 @@ CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) int64_t nCreationTime = GetTime(); CKeyMetadata metadata(nCreationTime); - // use HD key derivation if HD was enabled during wallet creation + // use HD key derivation if HD was enabled during wallet creation and a seed is present if (IsHDEnabled()) { DeriveNewChildKey(batch, metadata, secret, (CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); } else { @@ -283,6 +284,7 @@ bool CWallet::AddKeyPubKeyWithDB(WalletBatch &batch, const CKey& secret, const C secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); return true; } @@ -349,7 +351,11 @@ bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - return WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript); + if (WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript)) { + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + return true; + } + return false; } bool CWallet::LoadCScript(const CScript& redeemScript) @@ -374,7 +380,11 @@ bool CWallet::AddWatchOnly(const CScript& dest) const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - return WalletBatch(*database).WriteWatchOnly(dest, meta); + if (WalletBatch(*database).WriteWatchOnly(dest, meta)) { + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + return true; + } + return false; } bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) @@ -1402,6 +1412,7 @@ void CWallet::SetHDSeed(const CPubKey& seed) newHdChain.seed_id = seed.GetID(); SetHDChain(newHdChain, false); NotifyCanGetAddressesChanged(); + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); } void CWallet::SetHDChain(const CHDChain& chain, bool memonly) @@ -1418,6 +1429,30 @@ bool CWallet::IsHDEnabled() const return !hdChain.seed_id.IsNull(); } +bool CWallet::CanGenerateKeys() +{ + // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) + LOCK(cs_wallet); + return IsHDEnabled() || !CanSupportFeature(FEATURE_HD); +} + +bool CWallet::CanGetAddresses(bool internal) +{ + LOCK(cs_wallet); + // Check if the keypool has keys + bool keypool_has_keys; + if (internal && CanSupportFeature(FEATURE_HD_SPLIT)) { + keypool_has_keys = setInternalKeyPool.size() > 0; + } else { + keypool_has_keys = KeypoolCountExternalKeys() > 0; + } + // If the keypool doesn't have keys, check if we can generate them + if (!keypool_has_keys) { + return CanGenerateKeys(); + } + return keypool_has_keys; +} + void CWallet::SetWalletFlag(uint64_t flags) { LOCK(cs_wallet); @@ -1426,6 +1461,14 @@ void CWallet::SetWalletFlag(uint64_t flags) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } +void CWallet::UnsetWalletFlag(uint64_t flag) +{ + LOCK(cs_wallet); + m_wallet_flags &= ~flag; + if (!WalletBatch(*database).WriteWalletFlags(m_wallet_flags)) + throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); +} + bool CWallet::IsWalletFlagSet(uint64_t flag) { return (m_wallet_flags & flag); @@ -3103,7 +3146,8 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { LOCK(cs_KeyStore); // This wallet is in its first run if all of these are empty - fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() + && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); } if (nLoadWalletRet != DBErrors::LOAD_OK) @@ -3172,7 +3216,7 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s { bool fUpdated = false; { - LOCK(cs_wallet); // mapAddressBook + LOCK(cs_wallet); std::map<CTxDestination, CAddressBookData>::iterator mi = mapAddressBook.find(address); fUpdated = mi != mapAddressBook.end(); mapAddressBook[address].name = strName; @@ -3189,7 +3233,7 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s bool CWallet::DelAddressBook(const CTxDestination& address) { { - LOCK(cs_wallet); // mapAddressBook + LOCK(cs_wallet); // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); @@ -3288,7 +3332,7 @@ void CWallet::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) bool CWallet::TopUpKeyPool(unsigned int kpSize) { - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!CanGenerateKeys()) { return false; } { @@ -3418,7 +3462,7 @@ void CWallet::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey) bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) { - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!CanGetAddresses(internal)) { return false; } @@ -3619,6 +3663,10 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co bool CReserveKey::GetReservedKey(CPubKey& pubkey, bool internal) { + if (!pwallet->CanGetAddresses(internal)) { + return false; + } + if (nIndex == -1) { CKeyPool keypool; @@ -3869,7 +3917,6 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const { - LOCK(cs_wallet); std::vector<std::string> values; for (const auto& address : mapAddressBook) { for (const auto& data : address.second.destdata) { @@ -4073,14 +4120,16 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if ((wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { //selective allow to set flags walletInstance->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + } else if (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET) { + walletInstance->SetWalletFlag(WALLET_FLAG_BLANK_WALLET); } else { // generate a new seed CPubKey seed = walletInstance->GenerateNewSeed(); walletInstance->SetHDSeed(seed); - } + } // Otherwise, do not generate a new seed // Top up the keypool - if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !walletInstance->TopUpKeyPool()) { + if (walletInstance->CanGenerateKeys() && !walletInstance->TopUpKeyPool()) { InitError(_("Unable to generate initial keys")); return nullptr; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cf3fa0aced..c455b7cdad 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -136,9 +136,21 @@ enum WalletFlags : uint64_t { // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), + + //! Flag set when a wallet contains no HD seed and no private keys, scripts, + //! addresses, and other watch only things, and is therefore "blank." + //! + //! The only function this flag serves is to distinguish a blank wallet from + //! a newly created wallet when the wallet database is loaded, to avoid + //! initialization that should only happen on first run. + //! + //! This flag is also a mandatory flag to prevent previous versions of + //! bitcoin from opening the wallet, thinking it was newly created, and + //! then improperly reinitializing it. + WALLET_FLAG_BLANK_WALLET = (1ULL << 33), }; -static constexpr uint64_t g_known_wallet_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS; +static constexpr uint64_t g_known_wallet_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET; /** A key pool entry */ class CKeyPool @@ -788,7 +800,7 @@ public: int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; uint64_t nAccountingEntryNumber = 0; - std::map<CTxDestination, CAddressBookData> mapAddressBook; + std::map<CTxDestination, CAddressBookData> mapAddressBook GUARDED_BY(cs_wallet); std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); @@ -865,15 +877,15 @@ public: bool LoadCScript(const CScript& redeemScript); //! Adds a destination data tuple to the store, and saves it to disk - bool AddDestData(const CTxDestination &dest, const std::string &key, const std::string &value); + bool AddDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Erases a destination data tuple in the store and on disk - bool EraseDestData(const CTxDestination &dest, const std::string &key); + bool EraseDestData(const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a destination data tuple to the store, without saving it to disk - void LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value); + void LoadDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Look up a destination data tuple in the store, return true if found false otherwise - bool GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const; + bool GetDestData(const CTxDestination& dest, const std::string& key, std::string* value) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Get all destination values matching a prefix. - std::vector<std::string> GetDestValues(const std::string& prefix) const; + std::vector<std::string> GetDestValues(const std::string& prefix) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a watch-only address to the store, and saves it to disk. bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -1041,7 +1053,7 @@ public: bool DelAddressBook(const CTxDestination& address); - const std::string& GetLabelName(const CScript& scriptPubKey) const; + const std::string& GetLabelName(const CScript& scriptPubKey) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void GetScriptForMining(std::shared_ptr<CReserveScript> &script); @@ -1132,6 +1144,12 @@ public: /* Returns true if HD is enabled */ bool IsHDEnabled() const; + /* Returns true if the wallet can generate new keys */ + bool CanGenerateKeys(); + + /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ + bool CanGetAddresses(bool internal = false); + /* Generates a new HD seed (will not be activated) */ CPubKey GenerateNewSeed(); @@ -1169,6 +1187,9 @@ public: /** set a single wallet flag */ void SetWalletFlag(uint64_t flags); + /** Unsets a single wallet flag */ + void UnsetWalletFlag(uint64_t flag); + /** check if a certain wallet flag is set */ bool IsWalletFlagSet(uint64_t flag); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 30b0c48eef..628f3fe803 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -94,7 +94,6 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa static void WalletShowInfo(CWallet* wallet_instance) { - // lock required because of some AssertLockHeld() LOCK(wallet_instance->cs_wallet); fprintf(stdout, "Wallet info\n===========\n"); diff --git a/test/functional/README.md b/test/functional/README.md index 74f454b86c..5e3009e6af 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -43,6 +43,7 @@ don't have test cases for. - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` + - `tool` for tests for tools, eg `tool_wallet.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` - use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py new file mode 100755 index 0000000000..2cc5bc974b --- /dev/null +++ b/test/functional/rpc_deriveaddresses.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the deriveaddresses rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +class DeriveaddressesTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.supports_cli = 1 + + def run_test(self): + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, "a") + + descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + + assert_equal(self.nodes[0].deriveaddresses(descriptor), [address]) + + descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + + assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address]) + + ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)" + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 0, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + + assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)", 0, 2) + + assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)") + + assert_raises_rpc_error(-8, "Missing range end parameter", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", 0) + + assert_raises_rpc_error(-8, "Range end should be equal to or greater than begin", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", 2, 0) + + assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", -1, 0) + + combo_descriptor = "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)" + assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) + + hardened_without_privkey_descriptor = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)" + assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor) + + bare_multisig_descriptor = "multi(1, tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0, tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)" + assert_raises_rpc_error(-5, "Descriptor does not have a corresponding address", self.nodes[0].deriveaddresses, bare_multisig_descriptor) + +if __name__ == '__main__': + DeriveaddressesTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 1e10280e60..a82a5d0208 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -269,6 +269,9 @@ class PSBTTest(BitcoinTestFramework): combined = self.nodes[2].combinepsbt(combiner['combine']) assert_equal(combined, combiner['result']) + # Empty combiner test + assert_raises_rpc_error(-8, "Parameter 'txs' cannot be empty", self.nodes[0].combinepsbt, []) + # Finalizer test for finalizer in finalizers: finalized = self.nodes[2].finalizepsbt(finalizer['finalize'], False)['psbt'] diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d6e31457d1..999ea68254 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -209,12 +209,15 @@ class TestNode(): raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: - self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) - self.rpc.getblockcount() + rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up + self.log.debug("RPC successfully started") + if self.use_cli: + return + self.rpc = rpc self.rpc_connected = True self.url = self.rpc.url - self.log.debug("RPC successfully started") return except IOError as e: if e.errno != errno.ECONNREFUSED: # Port not yet open? diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 5c92370b85..d8f92e2601 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -7,8 +7,6 @@ This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. -Functional tests are disabled on Windows by default. Use --force to run them anyway. - For a description of arguments recognized by test scripts, see `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -118,8 +116,8 @@ BASE_SCRIPTS = [ 'mempool_persist.py', 'wallet_multiwallet.py', 'wallet_multiwallet.py --usecli', - 'wallet_disableprivatekeys.py', - 'wallet_disableprivatekeys.py --usecli', + 'wallet_createwallet.py', + 'wallet_createwallet.py --usecli', 'interface_http.py', 'interface_rpc.py', 'rpc_psbt.py', @@ -182,6 +180,8 @@ BASE_SCRIPTS = [ 'feature_filelock.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', + 'rpc_deriveaddresses.py', + 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', @@ -224,7 +224,6 @@ def main(): parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') - parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') @@ -261,12 +260,6 @@ def main(): enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - if config["environment"]["EXEEXT"] == ".exe" and not args.force: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Tests currently disabled on Windows by default. Use --force option to enable") - sys.exit(0) - if not enable_bitcoind: print("No functional tests to run.") print("Rerun ./configure with --with-daemon and then make") diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py new file mode 100755 index 0000000000..9fd2650d78 --- /dev/null +++ b/test/functional/wallet_createwallet.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test createwallet arguments. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +class CreateWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + self.supports_cli = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + node.generate(1) # Leave IBD for sethdseed + + self.nodes[0].createwallet(wallet_name='w0') + w0 = node.get_wallet_rpc('w0') + address1 = w0.getnewaddress() + + self.log.info("Test disableprivatekeys creation.") + self.nodes[0].createwallet(wallet_name='w1', disable_private_keys=True) + w1 = node.get_wallet_rpc('w1') + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w1.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) + w1.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info('Test that private keys cannot be imported') + addr = w0.getnewaddress('', 'legacy') + privkey = w0.dumpprivkey(addr) + assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) + result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) + assert(not result[0]['success']) + assert('warning' not in result[0]) + assert_equal(result[0]['error']['code'], -4) + assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') + + self.log.info("Test blank creation with private keys disabled.") + self.nodes[0].createwallet(wallet_name='w2', disable_private_keys=True, blank=True) + w2 = node.get_wallet_rpc('w2') + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w2.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w2.getrawchangeaddress) + w2.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info("Test blank creation with private keys enabled.") + self.nodes[0].createwallet(wallet_name='w3', disable_private_keys=False, blank=True) + w3 = node.get_wallet_rpc('w3') + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getrawchangeaddress) + # Import private key + w3.importprivkey(w0.dumpprivkey(address1)) + # Imported private keys are currently ignored by the keypool + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + # Set the seed + w3.sethdseed() + assert_equal(w3.getwalletinfo()['keypoolsize'], 1) + w3.getnewaddress() + w3.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys enabled and then encryption") + self.nodes[0].createwallet(wallet_name='w4', disable_private_keys=False, blank=True) + w4 = node.get_wallet_rpc('w4') + assert_equal(w4.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Encrypt the wallet. Nothing should change about the keypool + w4.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Now set a seed and it should work. Wallet should also be encrypted + w4.walletpassphrase('pass', 2) + w4.sethdseed() + w4.getnewaddress() + w4.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys disabled and then encryption") + self.nodes[0].createwallet(wallet_name='w5', disable_private_keys=True, blank=True) + w5 = node.get_wallet_rpc('w5') + assert_equal(w5.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getrawchangeaddress) + # Encrypt the wallet + w5.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getrawchangeaddress) + +if __name__ == '__main__': + CreateWalletTest().main() diff --git a/test/functional/wallet_disableprivatekeys.py b/test/functional/wallet_disableprivatekeys.py deleted file mode 100755 index e55bb82e76..0000000000 --- a/test/functional/wallet_disableprivatekeys.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test disable-privatekeys mode. -""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, -) - - -class DisablePrivateKeysTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = False - self.num_nodes = 1 - self.supports_cli = True - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def run_test(self): - node = self.nodes[0] - self.log.info("Test disableprivatekeys creation.") - self.nodes[0].createwallet('w1', True) - self.nodes[0].createwallet('w2') - w1 = node.get_wallet_rpc('w1') - w2 = node.get_wallet_rpc('w2') - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getnewaddress) - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) - w1.importpubkey(w2.getaddressinfo(w2.getnewaddress())['pubkey']) - - self.log.info('Test that private keys cannot be imported') - addr = w2.getnewaddress('', 'legacy') - privkey = w2.dumpprivkey(addr) - assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) - result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) - assert(not result[0]['success']) - assert('warning' not in result[0]) - assert_equal(result[0]['error']['code'], -4) - assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') - -if __name__ == '__main__': - DisablePrivateKeysTest().main() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index f122f19e3a..7cce72b39f 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -203,7 +203,7 @@ class ImportMultiTest(BitcoinTestFramework): "keys": [key.privkey]}, success=False, error_code=-4, - error_message='The wallet already contains the private key for this address or script') + error_message='The wallet already contains the private key for this address or script ("' + key.p2pkh_script + '")') # Address + Private key + watchonly self.log.info("Should import an address with private key and with watchonly") @@ -543,5 +543,88 @@ class ImportMultiTest(BitcoinTestFramework): solvable=True, ismine=False) + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") + self.test_importmulti({"desc": "sh(wpkh(" + key.pubkey + "))", + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=True, + label="Descriptor import test") + + # Test ranged descriptor fails if range is not specified + xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" + addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' + desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" + self.log.info("Ranged descriptor import should fail without a specified range") + self.test_importmulti({"desc": desc, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Descriptor is ranged, please specify the range') + + # Test importing of a ranged descriptor without keys + self.log.info("Should import the ranged descriptor with specified range as solvable") + self.test_importmulti({"desc": desc, + "timestamp": "now", + "range": {"end": 1}}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + for address in addresses: + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True) + + # Test importing of a P2PKH address via descriptor + key = get_key(self.nodes[0]) + self.log.info("Should import a p2pkh address from descriptor") + self.test_importmulti({"desc": "pkh(" + key.pubkey + ")", + "timestamp": "now", + "label": "Descriptor import test"}, + True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + solvable=True, + ismine=False, + label="Descriptor import test") + + # Test import fails if both desc and scriptPubKey are provided + key = get_key(self.nodes[0]) + self.log.info("Import should fail if both scriptPubKey and desc are provided") + self.test_importmulti({"desc": "pkh(" + key.pubkey + ")", + "scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Both a descriptor and a scriptPubKey should not be provided.') + + # Test import fails if neither desc nor scriptPubKey are present + key = get_key(self.nodes[0]) + self.log.info("Import should fail if neither a descriptor nor a scriptPubKey are provided") + self.test_importmulti({"timestamp": "now"}, + success=False, + error_code=-8, + error_message='Either a descriptor or scriptPubKey must be provided.') + + # Test importing of a multisig via descriptor + key1 = get_key(self.nodes[0]) + key2 = get_key(self.nodes[0]) + self.log.info("Should import a 1-of-2 bare multisig from descriptor") + self.test_importmulti({"desc": "multi(1," + key1.pubkey + "," + key2.pubkey + ")", + "timestamp": "now"}, + success=True) + self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") + test_address(self.nodes[1], + key1.p2pkh_addr, + ismine=False, + iswatchonly=False) + + if __name__ == '__main__': ImportMultiTest().main() diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index f0107f1361..d55119b266 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -4,7 +4,6 @@ bool:wallet/wallet.cpp float-divide-by-zero:policy/fees.cpp float-divide-by-zero:validation.cpp float-divide-by-zero:wallet/wallet.cpp -nonnull-attribute:support/cleanse.cpp unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h |