diff options
55 files changed, 1046 insertions, 560 deletions
diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index 139c03181f..6dc277f298 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -4,7 +4,9 @@ Utility to generate the seeds.txt list that is compiled into the client (see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and other utilities in [contrib/seeds](/contrib/seeds)). Be sure to update `PATTERN_AGENT` in `makeseeds.py` to include the current version, -and remove old versions as necessary. +and remove old versions as necessary (at a minimum when GetDesireableServiceFlags +changes its default return value, as those are the services which seeds are added +to addrman with). The seeds compiled into the release are created from sipa's DNS seed data, like this: diff --git a/doc/benchmarking.md b/doc/benchmarking.md index 0ba75afcda..99d36cb226 100644 --- a/doc/benchmarking.md +++ b/doc/benchmarking.md @@ -4,24 +4,49 @@ Benchmarking Bitcoin Core has an internal benchmarking framework, with benchmarks for cryptographic algorithms such as SHA1, SHA256, SHA512 and RIPEMD160. As well as the rolling bloom filter. +Running +--------------------- After compiling bitcoin-core, the benchmarks can be run with: -`src/bench/bench_bitcoin` + + src/bench/bench_bitcoin The output will look similar to: ``` -#Benchmark,count,min,max,average -RIPEMD160,448,0.001245033173334,0.002638196945190,0.002461894814457 -RollingBloom-refresh,1,0.000635000000000,0.000635000000000,0.000635000000000 -RollingBloom-refresh,1,0.000108000000000,0.000108000000000,0.000108000000000 -RollingBloom-refresh,1,0.000107000000000,0.000107000000000,0.000107000000000 -RollingBloom-refresh,1,0.000204000000000,0.000204000000000,0.000204000000000 -SHA1,640,0.000909024336207,0.001938136418660,0.001843086257577 -SHA256,256,0.002209486499909,0.008500099182129,0.004300644621253 -SHA512,384,0.001319904176016,0.002813005447388,0.002615700786312 -Sleep100ms,10,0.205592155456543,0.210056066513062,0.104166316986084 -Trig,67108864,0.000000014997003,0.000000015448112,0.000000015188842 +# Benchmark, evals, iterations, total, min, max, median +Base58CheckEncode, 5, 320000, 120.772, 7.49351e-05, 7.59374e-05, 7.54759e-05 +Base58Decode, 5, 800000, 122.833, 3.0467e-05, 3.11732e-05, 3.06304e-05 +Base58Encode, 5, 470000, 137.094, 5.81061e-05, 5.85109e-05, 5.84462e-05 +BenchLockedPool, 5, 530, 34.2023, 0.0128247, 0.0129613, 0.0129026 +CCheckQueueSpeedPrevectorJob, 5, 1400, 26.1762, 0.00365048, 0.00388629, 0.00367108 +CCoinsCaching, 5, 170000, 48.1074, 5.60229e-05, 5.72316e-05, 5.66214e-05 +CoinSelection, 5, 650, 34.6426, 0.0105801, 0.0107699, 0.010664 +DeserializeAndCheckBlockTest, 5, 160, 39.2084, 0.0483662, 0.0494199, 0.0490138 +DeserializeBlockTest, 5, 130, 23.8129, 0.0357731, 0.0373763, 0.0365858 +FastRandom_1bit, 5, 440000000, 38.1609, 1.72974e-08, 1.73882e-08, 1.73478e-08 +FastRandom_32bit, 5, 110000000, 72.8237, 1.29992e-07, 1.37014e-07, 1.30115e-07 +MempoolEviction, 5, 41000, 89.8883, 0.000432748, 0.000446857, 0.000438483 +PrevectorClear, 5, 5600, 47.9229, 0.00169952, 0.0017455, 0.00170315 +PrevectorDestructor, 5, 5700, 44.5498, 0.0015561, 0.00156977, 0.00156469 +RIPEMD160, 5, 440, 135.988, 0.0615496, 0.062268, 0.0617779 +RollingBloom, 5, 1500000, 36.5109, 4.80961e-06, 4.97463e-06, 4.85811e-06 +SHA1, 5, 570, 51.808, 0.018065, 0.0182623, 0.0181865 +SHA256, 5, 340, 8.31841, 0.00483231, 0.00499803, 0.00485486 +SHA256_32b, 5, 4700000, 10.469, 4.43441e-07, 4.47611e-07, 4.45223e-07 +SHA512, 5, 330, 33.3408, 0.02017, 0.0202554, 0.0201921 +SipHash_32b, 5, 40000000, 38.7088, 1.91103e-07, 1.96998e-07, 1.93792e-07 +Sleep100ms, 5, 10, 5.01062, 0.100131, 0.100368, 0.100147 +Trig, 5, 12000000, 5.95494, 9.78115e-08, 1.04354e-07, 9.80682e-08 +VerifyScriptBench, 5, 6300, 9.02493, 0.000285566, 0.000288433, 0.000286175 ``` +Help +--------------------- +`-?` will print a list of options and exit: + + src/bench/bench_bitcoin -? + +Notes +--------------------- More benchmarks are needed for, in no particular order: - Script Validation - CCoinDBView caching diff --git a/doc/release-notes.md b/doc/release-notes.md index d92666da72..29377d4a1d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -75,19 +75,26 @@ There is a checkbox to mark the transaction as final. The RPC default remains unchanged: to use RBF, launch with `-walletrbf=1` or use the `replaceable` argument for individual transactions. -Custom wallet directories ---------------------- -The ability to specify a directory other than the default data directory in which to store -wallets has been added. An existing directory can be specified using the `-walletdir=<dir>` -argument. Wallets loaded via `-wallet` arguments must be in this wallet directory. Care should be taken -when choosing a wallet directory location, as if it becomes unavailable during operation, -funds may be lost. - -Default wallet directory change --------------------------- -On new installations (if the data directory doesn't exist), wallets will now be stored in a -new `wallets/` subdirectory inside the data directory. If this `wallets/` subdirectory -doesn't exist (i.e. on existing nodes), the current datadir root is used instead, as it was. +Wallets directory configuration (`-walletdir`) +---------------------------------------------- + +Bitcoin Core now has more flexibility in where the wallets directory can be +located. Previously wallet database files were stored at the top level of the +bitcoin data directory. The behavior is now: + +- For new installations (where the data directory doesn't already exist), + wallets will now be stored in a new `wallets/` subdirectory inside the data + directory by default. +- For existing nodes (where the data directory already exists), wallets will be + stored in the data directory root by default. If a `wallets/` subdirectory + already exists in the data directory root, then wallets will be stored in the + `wallets/` subdirectory by default. +- The location of the wallets directory can be overridden by specifying a + `-walletdir=<path>` option where `<path>` can be an absolute path to a + directory or directory symlink. + +Care should be taken when choosing the wallets directory location, as if it +becomes unavailable during operation, funds may be lost. Low-level RPC changes ---------------------- @@ -97,6 +104,8 @@ Low-level RPC changes * `getwalletinfo` * `getmininginfo` - The wallet RPC `getreceivedbyaddress` will return an error if called with an address not in the wallet. +- The wallet RPC `addwitnessaddress` was deprecated and will be removed in version 0.17, + set the `address_type` argument of `getnewaddress`, or option `-addresstype=[bech32|p2sh-segwit]` instead. Changed command-line options ----------------------------- diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 97a6c346ab..39757baf9e 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -124,13 +124,17 @@ public: assert(consensus.hashGenesisBlock == uint256S("0x000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f")); assert(genesis.hashMerkleRoot == uint256S("0x4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")); - // Note that of those with the service bits flag, most only support a subset of possible options - vSeeds.emplace_back("seed.bitcoin.sipa.be", true); // Pieter Wuille, only supports x1, x5, x9, and xd - vSeeds.emplace_back("dnsseed.bluematt.me", true); // Matt Corallo, only supports x9 - vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org", false); // Luke Dashjr - vSeeds.emplace_back("seed.bitcoinstats.com", true); // Christian Decker, supports x1 - xf - vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch", true); // Jonas Schnelli, only supports x1, x5, x9, and xd - vSeeds.emplace_back("seed.btc.petertodd.org", true); // Peter Todd, only supports x1, x5, x9, and xd + // Note that of those which support the service bits prefix, most only support a subset of + // possible options. + // This is fine at runtime as we'll fall back to using them as a oneshot if they dont support the + // service bits we want, but we should get them updated to support all service bits wanted by any + // release ASAP to avoid it where possible. + vSeeds.emplace_back("seed.bitcoin.sipa.be"); // Pieter Wuille, only supports x1, x5, x9, and xd + vSeeds.emplace_back("dnsseed.bluematt.me"); // Matt Corallo, only supports x9 + vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org"); // Luke Dashjr + vSeeds.emplace_back("seed.bitcoinstats.com"); // Christian Decker, supports x1 - xf + vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch"); // Jonas Schnelli, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.btc.petertodd.org"); // Peter Todd, only supports x1, x5, x9, and xd base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,0); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,5); @@ -229,10 +233,10 @@ public: vFixedSeeds.clear(); vSeeds.clear(); // nodes with support for servicebits filtering should be at the top - vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch", true); - vSeeds.emplace_back("seed.tbtc.petertodd.org", true); - vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl", true); - vSeeds.emplace_back("testnet-seed.bluematt.me", false); + vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch"); + vSeeds.emplace_back("seed.tbtc.petertodd.org"); + vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl"); + vSeeds.emplace_back("testnet-seed.bluematt.me"); // Just a static list of stable node(s), only supports x9 base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196); diff --git a/src/chainparams.h b/src/chainparams.h index ecfe1a0ac3..d478da9891 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -14,12 +14,6 @@ #include <memory> #include <vector> -struct CDNSSeedData { - std::string host; - bool supportsServiceBitsFiltering; - CDNSSeedData(const std::string &strHost, bool supportsServiceBitsFilteringIn) : host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} -}; - struct SeedSpec6 { uint8_t addr[16]; uint16_t port; @@ -71,7 +65,8 @@ public: bool MineBlocksOnDemand() const { return fMineBlocksOnDemand; } /** Return the BIP70 network string (main, test or regtest) */ std::string NetworkIDString() const { return strNetworkID; } - const std::vector<CDNSSeedData>& DNSSeeds() const { return vSeeds; } + /** Return the list of hostnames to look up for DNS seeds */ + const std::vector<std::string>& DNSSeeds() const { return vSeeds; } const std::vector<unsigned char>& Base58Prefix(Base58Type type) const { return base58Prefixes[type]; } const std::string& Bech32HRP() const { return bech32_hrp; } const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } @@ -85,7 +80,7 @@ protected: CMessageHeader::MessageStartChars pchMessageStart; int nDefaultPort; uint64_t nPruneAfterHeight; - std::vector<CDNSSeedData> vSeeds; + std::vector<std::string> vSeeds; std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES]; std::string bech32_hrp; std::string strNetworkID; diff --git a/src/compat.h b/src/compat.h index 65e9683e2f..aae84b1181 100644 --- a/src/compat.h +++ b/src/compat.h @@ -33,7 +33,7 @@ #include <ws2tcpip.h> #include <stdint.h> #else -#include <sys/fcntl.h> +#include <fcntl.h> #include <sys/mman.h> #include <sys/select.h> #include <sys/socket.h> diff --git a/src/init.cpp b/src/init.cpp index 7215e87359..d1d733af94 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1143,23 +1143,10 @@ bool AppInitParameterInteraction() static bool LockDataDirectory(bool probeOnly) { - std::string strDataDir = GetDataDir().string(); - // Make sure only a single Bitcoin process is using the data directory. - fs::path pathLockFile = GetDataDir() / ".lock"; - FILE* file = fsbridge::fopen(pathLockFile, "a"); // empty lock file; created if it doesn't exist. - if (file) fclose(file); - - try { - static boost::interprocess::file_lock lock(pathLockFile.string().c_str()); - if (!lock.try_lock()) { - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), strDataDir, _(PACKAGE_NAME))); - } - if (probeOnly) { - lock.unlock(); - } - } catch(const boost::interprocess::interprocess_exception& e) { - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running.") + " %s.", strDataDir, _(PACKAGE_NAME), e.what())); + fs::path datadir = GetDataDir(); + if (!LockDirectory(datadir, ".lock", probeOnly)) { + return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), datadir.string(), _(PACKAGE_NAME))); } return true; } @@ -1223,6 +1210,15 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) LogPrintf("Using config file %s\n", GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string()); LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); + // Warn about relative -datadir path. + if (gArgs.IsArgSet("-datadir") && !fs::path(gArgs.GetArg("-datadir", "")).is_absolute()) { + LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " + "current working directory '%s'. This is fragile, because if bitcoin is started in the future " + "from a different location, it will be unable to locate the current data files. There could " + "also be data loss if bitcoin is started while in a temporary directory.\n", + gArgs.GetArg("-datadir", ""), fs::current_path().string()); + } + InitSignatureCache(); InitScriptExecutionCache(); diff --git a/src/miner.cpp b/src/miner.cpp index 4e63ab4df0..dda52790c6 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -352,7 +352,7 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda // Try to compare the mapTx entry to the mapModifiedTx entry iter = mempool.mapTx.project<0>(mi); if (modit != mapModifiedTx.get<ancestor_score>().end() && - CompareModifiedEntry()(*modit, CTxMemPoolModifiedEntry(iter))) { + CompareTxMemPoolEntryByAncestorFee()(*modit, CTxMemPoolModifiedEntry(iter))) { // The best entry in mapModifiedTx has higher score // than the one from mapTx. // Switch which transaction (package) to consider diff --git a/src/miner.h b/src/miner.h index 698b4a4788..9c086332d4 100644 --- a/src/miner.h +++ b/src/miner.h @@ -41,6 +41,12 @@ struct CTxMemPoolModifiedEntry { nSigOpCostWithAncestors = entry->GetSigOpCostWithAncestors(); } + int64_t GetModifiedFee() const { return iter->GetModifiedFee(); } + uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } + CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } + size_t GetTxSize() const { return iter->GetTxSize(); } + const CTransaction& GetTx() const { return iter->GetTx(); } + CTxMemPool::txiter iter; uint64_t nSizeWithAncestors; CAmount nModFeesWithAncestors; @@ -67,21 +73,6 @@ struct modifiedentry_iter { } }; -// This matches the calculation in CompareTxMemPoolEntryByAncestorFee, -// except operating on CTxMemPoolModifiedEntry. -// TODO: refactor to avoid duplication of this logic. -struct CompareModifiedEntry { - bool operator()(const CTxMemPoolModifiedEntry &a, const CTxMemPoolModifiedEntry &b) const - { - double f1 = (double)a.nModFeesWithAncestors * b.nSizeWithAncestors; - double f2 = (double)b.nModFeesWithAncestors * a.nSizeWithAncestors; - if (f1 == f2) { - return CTxMemPool::CompareIteratorByHash()(a.iter, b.iter); - } - return f1 > f2; - } -}; - // A comparator that sorts transactions based on number of ancestors. // This is sufficient to sort an ancestor package in an order that is valid // to appear in a block. @@ -106,7 +97,7 @@ typedef boost::multi_index_container< // Reuse same tag from CTxMemPool's similar index boost::multi_index::tag<ancestor_score>, boost::multi_index::identity<CTxMemPoolModifiedEntry>, - CompareModifiedEntry + CompareTxMemPoolEntryByAncestorFee > > > indexed_modified_transaction_set; diff --git a/src/net.cpp b/src/net.cpp index 682f47bff2..8111390749 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -136,7 +136,7 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn for (const auto& seed_in : vSeedsIn) { struct in6_addr ip; memcpy(&ip, seed_in.addr, sizeof(ip)); - CAddress addr(CService(ip, seed_in.port), NODE_NETWORK); + CAddress addr(CService(ip, seed_in.port), GetDesirableServiceFlags(NODE_NONE)); addr.nTime = GetTime() - GetRand(nOneWeek) - nOneWeek; vSeedsOut.push_back(addr); } @@ -1577,19 +1577,6 @@ void MapPort(bool) -static std::string GetDNSHost(const CDNSSeedData& data, ServiceFlags* requiredServiceBits) -{ - //use default host for non-filter-capable seeds or if we use the default service bits (NODE_NETWORK) - if (!data.supportsServiceBitsFiltering || *requiredServiceBits == NODE_NETWORK) { - *requiredServiceBits = NODE_NETWORK; - return data.host; - } - - // See chainparams.cpp, most dnsseeds only support one or two possible servicebits hostnames - return strprintf("x%x.%s", *requiredServiceBits, data.host); -} - - void CConnman::ThreadDNSAddressSeed() { // goal: only query DNS seeds if address need is acute @@ -1612,22 +1599,22 @@ void CConnman::ThreadDNSAddressSeed() } } - const std::vector<CDNSSeedData> &vSeeds = Params().DNSSeeds(); + const std::vector<std::string> &vSeeds = Params().DNSSeeds(); int found = 0; LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); - for (const CDNSSeedData &seed : vSeeds) { + for (const std::string &seed : vSeeds) { if (interruptNet) { return; } if (HaveNameProxy()) { - AddOneShot(seed.host); + AddOneShot(seed); } else { std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); - std::string host = GetDNSHost(seed, &requiredServiceBits); + std::string host = strprintf("x%x.%s", requiredServiceBits, seed); CNetAddr resolveSource; if (!resolveSource.SetInternal(host)) { continue; @@ -1643,6 +1630,10 @@ void CConnman::ThreadDNSAddressSeed() found++; } addrman.Add(vAdd, resolveSource); + } else { + // We now avoid directly using results from DNS Seeds which do not support service bit filtering, + // instead using them as a oneshot to get nodes with our desired service bits. + AddOneShot(seed); } } } diff --git a/src/protocol.h b/src/protocol.h index 452b5b28ce..43d8ac067a 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -291,7 +291,15 @@ enum ServiceFlags : uint64_t { * unless they set NODE_NETWORK_LIMITED and we are out of IBD, in which * case NODE_NETWORK_LIMITED suffices). * - * Thus, generally, avoid calling with peerServices == NODE_NONE. + * Thus, generally, avoid calling with peerServices == NODE_NONE, unless + * state-specific flags must absolutely be avoided. When called with + * peerServices == NODE_NONE, the returned desirable service flags are + * guaranteed to not change dependant on state - ie they are suitable for + * use when describing peers which we know to be desirable, but for which + * we do not have a confirmed set of service flags. + * + * If the NODE_NONE return value is changed, contrib/seeds/makeseeds.py + * should be updated appropriately to filter for the same nodes. */ static ServiceFlags GetDesirableServiceFlags(ServiceFlags services) { return ServiceFlags(NODE_NETWORK | NODE_WITNESS); diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index a2521a1e9e..74f0db3520 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -341,7 +341,7 @@ void AddressTableModel::updateEntry(const QString &address, priv->updateEntry(address, label, isMine, purpose, status); } -QString AddressTableModel::addRow(const QString &type, const QString &label, const QString &address) +QString AddressTableModel::addRow(const QString &type, const QString &label, const QString &address, const OutputType address_type) { std::string strLabel = label.toStdString(); std::string strAddress = address.toStdString(); @@ -384,8 +384,8 @@ QString AddressTableModel::addRow(const QString &type, const QString &label, con return QString(); } } - wallet->LearnRelatedScripts(newKey, g_address_type); - strAddress = EncodeDestination(GetDestinationForKey(newKey, g_address_type)); + wallet->LearnRelatedScripts(newKey, address_type); + strAddress = EncodeDestination(GetDestinationForKey(newKey, address_type)); } else { diff --git a/src/qt/addresstablemodel.h b/src/qt/addresstablemodel.h index d04b95ebae..11439e25d5 100644 --- a/src/qt/addresstablemodel.h +++ b/src/qt/addresstablemodel.h @@ -8,6 +8,8 @@ #include <QAbstractTableModel> #include <QStringList> +enum OutputType : int; + class AddressTablePriv; class WalletModel; @@ -61,7 +63,7 @@ public: /* Add an address to the model. Returns the added address on success, and an empty string otherwise. */ - QString addRow(const QString &type, const QString &label, const QString &address); + QString addRow(const QString &type, const QString &label, const QString &address, const OutputType address_type); /* Look up label for address in address book, if not found return empty string. */ diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index 6b7df2f2b9..a945fc6aa0 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -11,6 +11,8 @@ #include <QDataWidgetMapper> #include <QMessageBox> +extern OutputType g_address_type; + EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : QDialog(parent), ui(new Ui::EditAddressDialog), @@ -77,7 +79,8 @@ bool EditAddressDialog::saveCurrentRow() address = model->addRow( mode == NewSendingAddress ? AddressTableModel::Send : AddressTableModel::Receive, ui->labelEdit->text(), - ui->addressEdit->text()); + ui->addressEdit->text(), + g_address_type); break; case EditReceivingAddress: case EditSendingAddress: diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 58f030ebf0..09fb435a58 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>776</width> - <height>364</height> + <height>396</height> </rect> </property> <layout class="QVBoxLayout" name="verticalLayout" stretch="0,0,1"> @@ -28,6 +28,22 @@ <layout class="QVBoxLayout" name="verticalLayout_3"> <item> <layout class="QGridLayout" name="gridLayout"> + <item row="5" column="0"> + <widget class="QLabel" name="label"> + <property name="toolTip"> + <string>An optional amount to request. Leave this empty or zero to not request a specific amount.</string> + </property> + <property name="text"> + <string>&Amount:</string> + </property> + <property name="alignment"> + <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> + </property> + <property name="buddy"> + <cstring>reqAmount</cstring> + </property> + </widget> + </item> <item row="6" column="0"> <widget class="QLabel" name="label_3"> <property name="toolTip"> @@ -51,13 +67,6 @@ </property> </widget> </item> - <item row="6" column="2"> - <widget class="QLineEdit" name="reqMessage"> - <property name="toolTip"> - <string>An optional message to attach to the payment request, which will be displayed when the request is opened. Note: The message will not be sent with the payment over the Bitcoin network.</string> - </property> - </widget> - </item> <item row="2" column="2"> <widget class="QLabel" name="label_5"> <property name="text"> @@ -81,32 +90,10 @@ </property> </widget> </item> - <item row="5" column="0"> - <widget class="QLabel" name="label"> - <property name="toolTip"> - <string>An optional amount to request. Leave this empty or zero to not request a specific amount.</string> - </property> - <property name="text"> - <string>&Amount:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>reqAmount</cstring> - </property> - </widget> - </item> - <item row="5" column="2"> - <widget class="BitcoinAmountField" name="reqAmount"> - <property name="minimumSize"> - <size> - <width>80</width> - <height>0</height> - </size> - </property> + <item row="6" column="2"> + <widget class="QLineEdit" name="reqMessage"> <property name="toolTip"> - <string>An optional amount to request. Leave this empty or zero to not request a specific amount.</string> + <string>An optional message to attach to the payment request, which will be displayed when the request is opened. Note: The message will not be sent with the payment over the Bitcoin network.</string> </property> </widget> </item> @@ -174,6 +161,73 @@ </property> </widget> </item> + <item row="5" column="2"> + <layout class="QHBoxLayout" name="horizontalLayout_6"> + <item> + <widget class="BitcoinAmountField" name="reqAmount"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="minimumSize"> + <size> + <width>80</width> + <height>0</height> + </size> + </property> + <property name="maximumSize"> + <size> + <width>1000</width> + <height>100</height> + </size> + </property> + <property name="toolTip"> + <string>An optional amount to request. Leave this empty or zero to not request a specific amount.</string> + </property> + </widget> + </item> + <item> + <widget class="QCheckBox" name="useBech32"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="maximumSize"> + <size> + <width>1000</width> + <height>16777215</height> + </size> + </property> + <property name="focusPolicy"> + <enum>Qt::StrongFocus</enum> + </property> + <property name="toolTip"> + <string>Bech32 addresses (BIP-173) are cheaper to spend from and offer better protection against typos. When unchecked a P2SH wrapped SegWit address will be created, compatible with older wallets.</string> + </property> + <property name="text"> + <string>Generate Bech32 address</string> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer_3"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>0</width> + <height>0</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> </layout> </item> </layout> @@ -306,6 +360,7 @@ <tabstops> <tabstop>reqLabel</tabstop> <tabstop>reqAmount</tabstop> + <tabstop>useBech32</tabstop> <tabstop>reqMessage</tabstop> <tabstop>receiveButton</tabstop> <tabstop>clearButton</tabstop> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 670d6108db..558d4f108c 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -995,6 +995,18 @@ QString formatBytes(uint64_t bytes) return QString(QObject::tr("%1 GB")).arg(bytes / 1024 / 1024 / 1024); } +qreal calculateIdealFontSize(int width, const QString& text, QFont font, qreal minPointSize, qreal font_size) { + while(font_size >= minPointSize) { + font.setPointSizeF(font_size); + QFontMetrics fm(font); + if (fm.width(text) < width) { + break; + } + font_size -= 0.5; + } + return font_size; +} + void ClickableLabel::mouseReleaseEvent(QMouseEvent *event) { Q_EMIT clicked(event->pos()); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index ad0e22ccd6..71a69483f5 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -201,6 +201,8 @@ namespace GUIUtil QString formatBytes(uint64_t bytes); + qreal calculateIdealFontSize(int width, const QString& text, QFont font, qreal minPointSize = 4, qreal startPointSize = 14); + class ClickableLabel : public QLabel { Q_OBJECT diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index e07f2b74ee..7fd5285467 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <wallet/wallet.h> + #include <qt/receivecoinsdialog.h> #include <qt/forms/ui_receivecoinsdialog.h> @@ -91,6 +93,15 @@ void ReceiveCoinsDialog::setModel(WalletModel *_model) SLOT(recentRequestsView_selectionChanged(QItemSelection, QItemSelection))); // Last 2 columns are set by the columnResizingFixer, when the table geometry is ready. columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(tableView, AMOUNT_MINIMUM_COLUMN_WIDTH, DATE_COLUMN_WIDTH, this); + + // configure bech32 checkbox, disable if launched with legacy as default: + if (model->getDefaultAddressType() == OUTPUT_TYPE_BECH32) { + ui->useBech32->setCheckState(Qt::Checked); + } else { + ui->useBech32->setCheckState(Qt::Unchecked); + } + + ui->useBech32->setVisible(model->getDefaultAddressType() != OUTPUT_TYPE_LEGACY); } } @@ -133,7 +144,11 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() QString address; QString label = ui->reqLabel->text(); /* Generate new receiving address */ - address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); + OutputType address_type = model->getDefaultAddressType(); + if (address_type != OUTPUT_TYPE_LEGACY) { + address_type = ui->useBech32->isChecked() ? OUTPUT_TYPE_BECH32 : OUTPUT_TYPE_P2SH_SEGWIT; + } + address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, "", address_type); SendCoinsRecipient info(address, label, ui->reqAmount->value(), ui->reqMessage->text()); ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index 209397ca0c..d4cb0e5ba2 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -183,9 +183,13 @@ void ReceiveRequestDialog::update() QPainter painter(&qrAddrImage); painter.drawImage(0, 0, qrImage.scaled(QR_IMAGE_SIZE, QR_IMAGE_SIZE)); QFont font = GUIUtil::fixedPitchFont(); - font.setPixelSize(12); - painter.setFont(font); QRect paddedRect = qrAddrImage.rect(); + + // calculate ideal font size + qreal font_size = GUIUtil::calculateIdealFontSize(paddedRect.width() - 20, info.address, font); + font.setPointSizeF(font_size); + + painter.setFont(font); paddedRect.setHeight(QR_IMAGE_SIZE+12); painter.drawText(paddedRect, Qt::AlignBottom|Qt::AlignCenter, info.address); painter.end(); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index a270e5de59..cd49292138 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -169,7 +169,9 @@ void TestGUI() } { LOCK(cs_main); - wallet.ScanForWalletTransactions(chainActive.Genesis(), nullptr, true); + WalletRescanReserver reserver(&wallet); + reserver.reserve(); + wallet.ScanForWalletTransactions(chainActive.Genesis(), nullptr, reserver, true); } wallet.SetBroadcastTransactions(true); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 494b46905d..4d7e977fcb 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -733,6 +733,11 @@ bool WalletModel::hdEnabled() const return wallet->IsHDEnabled(); } +OutputType WalletModel::getDefaultAddressType() const +{ + return g_address_type; +} + int WalletModel::getDefaultConfirmTarget() const { return nTxConfirmTarget; diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 7d7e4f160e..9e13de79be 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -15,6 +15,8 @@ #include <QObject> +enum OutputType : int; + class AddressTableModel; class OptionsModel; class PlatformStyle; @@ -214,6 +216,8 @@ public: bool hdEnabled() const; + OutputType getDefaultAddressType() const; + int getDefaultConfirmTarget() const; private: diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1b2c71c4a4..346672e45a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -24,6 +24,7 @@ #include <util.h> #include <utilstrencodings.h> #include <hash.h> +#include <validationinterface.h> #include <warnings.h> #include <stdint.h> @@ -323,6 +324,21 @@ UniValue waitforblockheight(const JSONRPCRequest& request) return ret; } +UniValue syncwithvalidationinterfacequeue(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 0) { + throw std::runtime_error( + "syncwithvalidationinterfacequeue\n" + "\nWaits for the validation interface queue to catch up on everything that was there when we entered this function.\n" + "\nExamples:\n" + + HelpExampleCli("syncwithvalidationinterfacequeue","") + + HelpExampleRpc("syncwithvalidationinterfacequeue","") + ); + } + SyncWithValidationInterfaceQueue(); + return NullUniValue; +} + UniValue getdifficulty(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) @@ -1628,6 +1644,7 @@ static const CRPCCommand commands[] = { "hidden", "waitfornewblock", &waitfornewblock, {"timeout"} }, { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, + { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, }; void RegisterBlockchainRPCCommands(CRPCTable &t) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 62bcdea31c..c11dda22c4 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -623,6 +623,18 @@ UniValue echo(const JSONRPCRequest& request) return request.params; } +static UniValue getinfo_deprecated(const JSONRPCRequest& request) +{ + throw JSONRPCError(RPC_METHOD_NOT_FOUND, + "getinfo\n" + "\nThis call was removed in version 0.16.0. Use the appropriate fields from:\n" + "- getblockchaininfo: blocks, difficulty, chain\n" + "- getnetworkinfo: version, protocolversion, timeoffset, connections, proxy, relayfee, warnings\n" + "- getwalletinfo: balance, keypoololdest, keypoolsize, paytxfee, unlocked_until, walletversion\n" + "\nbitcoin-cli has the option -getinfo to collect and format these in the old format." + ); +} + static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- @@ -637,6 +649,7 @@ static const CRPCCommand commands[] = { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, { "hidden", "echojson", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "getinfo", &getinfo_deprecated, {}}, }; void RegisterMiscRPCCommands(CRPCTable &t) diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index f56f341498..1766c6a093 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -287,35 +287,6 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx()); pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx()); - /* Now check the sort on the mining score index. - * Final order should be: - * - * tx7 (2M) - * tx2 (20k) - * tx4 (15000) - * tx1/tx5 (10000) - * tx3/6 (0) - * (Ties resolved by hash) - */ - sortedOrder.clear(); - sortedOrder.push_back(tx7.GetHash().ToString()); - sortedOrder.push_back(tx2.GetHash().ToString()); - sortedOrder.push_back(tx4.GetHash().ToString()); - if (tx1.GetHash() < tx5.GetHash()) { - sortedOrder.push_back(tx5.GetHash().ToString()); - sortedOrder.push_back(tx1.GetHash().ToString()); - } else { - sortedOrder.push_back(tx1.GetHash().ToString()); - sortedOrder.push_back(tx5.GetHash().ToString()); - } - if (tx3.GetHash() < tx6.GetHash()) { - sortedOrder.push_back(tx6.GetHash().ToString()); - sortedOrder.push_back(tx3.GetHash().ToString()); - } else { - sortedOrder.push_back(tx3.GetHash().ToString()); - sortedOrder.push_back(tx6.GetHash().ToString()); - } - CheckSort<mining_score>(pool, sortedOrder); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) @@ -427,6 +398,23 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) sortedOrder.erase(sortedOrder.end()-2); sortedOrder.insert(sortedOrder.begin(), tx7.GetHash().ToString()); CheckSort<ancestor_score>(pool, sortedOrder); + + // High-fee parent, low-fee child + // tx7 -> tx8 + CMutableTransaction tx8 = CMutableTransaction(); + tx8.vin.resize(1); + tx8.vin[0].prevout = COutPoint(tx7.GetHash(), 0); + tx8.vin[0].scriptSig = CScript() << OP_11; + tx8.vout.resize(1); + tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx8.vout[0].nValue = 10*COIN; + + // Check that we sort by min(feerate, ancestor_feerate): + // set the fee so that the ancestor feerate is above tx1/5, + // but the transaction's own feerate is lower + pool.addUnchecked(tx8.GetHash(), entry.Fee(5000LL).FromTx(tx8)); + sortedOrder.insert(sortedOrder.end()-1, tx8.GetHash().ToString()); + CheckSort<ancestor_score>(pool, sortedOrder); } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index ffb024aef9..d1edde284f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -906,8 +906,8 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); - // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + // Estimate the overhead of mapTx to be 12 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { diff --git a/src/txmempool.h b/src/txmempool.h index 512e70f8fa..d6f8e7094b 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -206,18 +206,14 @@ class CompareTxMemPoolEntryByDescendantScore public: bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { - bool fUseADescendants = UseDescendantScore(a); - bool fUseBDescendants = UseDescendantScore(b); + double a_mod_fee, a_size, b_mod_fee, b_size; - double aModFee = fUseADescendants ? a.GetModFeesWithDescendants() : a.GetModifiedFee(); - double aSize = fUseADescendants ? a.GetSizeWithDescendants() : a.GetTxSize(); - - double bModFee = fUseBDescendants ? b.GetModFeesWithDescendants() : b.GetModifiedFee(); - double bSize = fUseBDescendants ? b.GetSizeWithDescendants() : b.GetTxSize(); + GetModFeeAndSize(a, a_mod_fee, a_size); + GetModFeeAndSize(b, b_mod_fee, b_size); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). - double f1 = aModFee * bSize; - double f2 = aSize * bModFee; + double f1 = a_mod_fee * b_size; + double f2 = a_size * b_mod_fee; if (f1 == f2) { return a.GetTime() >= b.GetTime(); @@ -225,12 +221,21 @@ public: return f1 < f2; } - // Calculate which score to use for an entry (avoiding division). - bool UseDescendantScore(const CTxMemPoolEntry &a) const + // Return the fee/size we're using for sorting this entry. + void GetModFeeAndSize(const CTxMemPoolEntry &a, double &mod_fee, double &size) const { + // Compare feerate with descendants to feerate of the transaction, and + // return the fee/size for the max. double f1 = (double)a.GetModifiedFee() * a.GetSizeWithDescendants(); double f2 = (double)a.GetModFeesWithDescendants() * a.GetTxSize(); - return f2 > f1; + + if (f2 > f1) { + mod_fee = a.GetModFeesWithDescendants(); + size = a.GetSizeWithDescendants(); + } else { + mod_fee = a.GetModifiedFee(); + size = a.GetTxSize(); + } } }; @@ -261,33 +266,53 @@ public: } }; +/** \class CompareTxMemPoolEntryByAncestorScore + * + * Sort an entry by min(score/size of entry's tx, score/size with all ancestors). + */ class CompareTxMemPoolEntryByAncestorFee { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const + template<typename T> + bool operator()(const T& a, const T& b) const { - double aFees = a.GetModFeesWithAncestors(); - double aSize = a.GetSizeWithAncestors(); + double a_mod_fee, a_size, b_mod_fee, b_size; - double bFees = b.GetModFeesWithAncestors(); - double bSize = b.GetSizeWithAncestors(); + GetModFeeAndSize(a, a_mod_fee, a_size); + GetModFeeAndSize(b, b_mod_fee, b_size); // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). - double f1 = aFees * bSize; - double f2 = aSize * bFees; + double f1 = a_mod_fee * b_size; + double f2 = a_size * b_mod_fee; if (f1 == f2) { return a.GetTx().GetHash() < b.GetTx().GetHash(); } - return f1 > f2; } + + // Return the fee/size we're using for sorting this entry. + template <typename T> + void GetModFeeAndSize(const T &a, double &mod_fee, double &size) const + { + // Compare feerate with ancestors to feerate of the transaction, and + // return the fee/size for the min. + double f1 = (double)a.GetModifiedFee() * a.GetSizeWithAncestors(); + double f2 = (double)a.GetModFeesWithAncestors() * a.GetTxSize(); + + if (f1 > f2) { + mod_fee = a.GetModFeesWithAncestors(); + size = a.GetSizeWithAncestors(); + } else { + mod_fee = a.GetModifiedFee(); + size = a.GetTxSize(); + } + } }; // Multi_index tag names struct descendant_score {}; struct entry_time {}; -struct mining_score {}; struct ancestor_score {}; class CBlockPolicyEstimator; @@ -356,7 +381,6 @@ public: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] * - time in mempool - * - mining score (feerate modified by any fee deltas from PrioritiseTransaction) * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given @@ -446,12 +470,6 @@ public: boost::multi_index::identity<CTxMemPoolEntry>, CompareTxMemPoolEntryByEntryTime >, - // sorted by score (for mining prioritization) - boost::multi_index::ordered_unique< - boost::multi_index::tag<mining_score>, - boost::multi_index::identity<CTxMemPoolEntry>, - CompareTxMemPoolEntryByScore - >, // sorted by fee rate with ancestors boost::multi_index::ordered_non_unique< boost::multi_index::tag<ancestor_score>, diff --git a/src/util.cpp b/src/util.cpp index 150bc503df..80eed24ffd 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -72,6 +72,7 @@ #include <boost/algorithm/string/case_conv.hpp> // for to_lower() #include <boost/algorithm/string/predicate.hpp> // for startswith() and endswith() +#include <boost/interprocess/sync/file_lock.hpp> #include <boost/program_options/detail/config_file.hpp> #include <boost/thread.hpp> #include <openssl/crypto.h> @@ -375,6 +376,27 @@ int LogPrintStr(const std::string &str) return ret; } +bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only) +{ + fs::path pathLockFile = directory / lockfile_name; + FILE* file = fsbridge::fopen(pathLockFile, "a"); // empty lock file; created if it doesn't exist. + if (file) fclose(file); + + try { + static std::map<std::string, boost::interprocess::file_lock> locks; + boost::interprocess::file_lock& lock = locks.emplace(pathLockFile.string(), pathLockFile.string().c_str()).first->second; + if (!lock.try_lock()) { + return false; + } + if (probe_only) { + lock.unlock(); + } + } catch (const boost::interprocess::interprocess_exception& e) { + return error("Error while attempting to lock directory %s: %s", directory.string(), e.what()); + } + return true; +} + /** Interpret string as boolean, for argument parsing */ static bool InterpretBool(const std::string& strValue) { diff --git a/src/util.h b/src/util.h index 6a0d6a31e7..277b4c66af 100644 --- a/src/util.h +++ b/src/util.h @@ -173,6 +173,7 @@ bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length); bool RenameOver(fs::path src, fs::path dest); +bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); bool TryCreateDirectories(const fs::path& p); fs::path GetDefaultDataDir(); const fs::path &GetDataDir(bool fNetSpecific = true); diff --git a/src/validation.cpp b/src/validation.cpp index 14d60bb269..8cee0dfac3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1121,7 +1121,13 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - if (!ReadBlockFromDisk(block, pindex->GetBlockPos(), consensusParams)) + CDiskBlockPos blockPos; + { + LOCK(cs_main); + blockPos = pindex->GetBlockPos(); + } + + if (!ReadBlockFromDisk(block, blockPos, consensusParams)) return false; if (block.GetHash() != pindex->GetBlockHash()) return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 35ff0e1eec..23c6279128 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -95,7 +95,7 @@ void CDBEnv::Close() EnvShutdown(); } -bool CDBEnv::Open(const fs::path& pathIn) +bool CDBEnv::Open(const fs::path& pathIn, bool retry) { if (fDbEnvInit) return true; @@ -103,6 +103,11 @@ bool CDBEnv::Open(const fs::path& pathIn) boost::this_thread::interruption_point(); strPath = pathIn.string(); + if (!LockDirectory(pathIn, ".walletlock")) { + LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); + return false; + } + fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; @@ -134,7 +139,24 @@ bool CDBEnv::Open(const fs::path& pathIn) S_IRUSR | S_IWUSR); if (ret != 0) { dbenv->close(0); - return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); + LogPrintf("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret)); + if (retry) { + // try moving the database env out of the way + fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime()); + try { + fs::rename(pathLogDir, pathDatabaseBak); + LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string()); + } catch (const fs::filesystem_error&) { + // failure is ok (well, not really, but it's not worse than what we started with) + } + // try opening it again one more time + if (!Open(pathIn, false)) { + // if it still fails, it probably means we can't even create the database env + return false; + } + } else { + return false; + } } fDbEnvInit = true; @@ -269,25 +291,11 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walle return false; } - if (!bitdb.Open(walletDir)) - { - // try moving the database env out of the way - fs::path pathDatabase = walletDir / "database"; - fs::path pathDatabaseBak = walletDir / strprintf("database.%d.bak", GetTime()); - try { - fs::rename(pathDatabase, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); - } catch (const fs::filesystem_error&) { - // failure is ok (well, not really, but it's not worse than what we started with) - } - - // try again - if (!bitdb.Open(walletDir)) { - // if it still fails, it probably means we can't even create the database env - errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); - return false; - } + if (!bitdb.Open(walletDir, true)) { + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); + return false; } + return true; } diff --git a/src/wallet/db.h b/src/wallet/db.h index c6f317927f..787135e400 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -68,7 +68,7 @@ public: typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; bool Salvage(const std::string& strFile, bool fAggressive, std::vector<KeyValPair>& vResult); - bool Open(const fs::path& path); + bool Open(const fs::path& path, bool retry = 0); void Close(); void Flush(bool fShutdown); void CheckpointLSN(const std::string& strFile); diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 0f453f154f..2d26f7ae0f 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -205,11 +205,15 @@ bool VerifyWallets() return true; } - if (gArgs.IsArgSet("-walletdir") && !fs::is_directory(GetWalletDir())) { - if (fs::exists(fs::system_complete(gArgs.GetArg("-walletdir", "")))) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), gArgs.GetArg("-walletdir", "").c_str())); + if (gArgs.IsArgSet("-walletdir")) { + fs::path wallet_dir = gArgs.GetArg("-walletdir", ""); + if (!fs::exists(wallet_dir)) { + return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + } else if (!fs::is_directory(wallet_dir)) { + return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + } else if (!wallet_dir.is_absolute()) { + return InitError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); } - return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), gArgs.GetArg("-walletdir", "").c_str())); } LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 645e776b68..0b021f9fe0 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -86,7 +86,8 @@ UniValue importprivkey(const JSONRPCRequest& request) "1. \"privkey\" (string, required) The private key (see dumpprivkey)\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" "3. rescan (boolean, optional, default=true) Rescan the wallet for transactions\n" - "\nNote: This call can take minutes to complete if rescan is true.\n" + "\nNote: This call can take minutes to complete if rescan is true, during that time, other rpc calls\n" + "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" "\nExamples:\n" "\nDump a private key\n" + HelpExampleCli("dumpprivkey", "\"myaddress\"") + @@ -101,61 +102,65 @@ UniValue importprivkey(const JSONRPCRequest& request) ); - LOCK2(cs_main, pwallet->cs_wallet); - - EnsureWalletIsUnlocked(pwallet); - - std::string strSecret = request.params[0].get_str(); - std::string strLabel = ""; - if (!request.params[1].isNull()) - strLabel = request.params[1].get_str(); - - // Whether to perform rescan after import + WalletRescanReserver reserver(pwallet); bool fRescan = true; - if (!request.params[2].isNull()) - fRescan = request.params[2].get_bool(); - - if (fRescan && fPruneMode) - throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + { + LOCK2(cs_main, pwallet->cs_wallet); - CBitcoinSecret vchSecret; - bool fGood = vchSecret.SetString(strSecret); + EnsureWalletIsUnlocked(pwallet); - if (!fGood) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + std::string strSecret = request.params[0].get_str(); + std::string strLabel = ""; + if (!request.params[1].isNull()) + strLabel = request.params[1].get_str(); - CKey key = vchSecret.GetKey(); - if (!key.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Private key outside allowed range"); + // Whether to perform rescan after import + if (!request.params[2].isNull()) + fRescan = request.params[2].get_bool(); - CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); - CKeyID vchAddress = pubkey.GetID(); - { - pwallet->MarkDirty(); + if (fRescan && fPruneMode) + throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); - // We don't know which corresponding address will be used; label them all - for (const auto& dest : GetAllDestinationsForKey(pubkey)) { - pwallet->SetAddressBook(dest, strLabel, "receive"); + if (fRescan && !reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } - // Don't throw error in case a key is already there - if (pwallet->HaveKey(vchAddress)) { - return NullUniValue; - } + CBitcoinSecret vchSecret; + bool fGood = vchSecret.SetString(strSecret); - pwallet->mapKeyMetadata[vchAddress].nCreateTime = 1; + if (!fGood) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); - if (!pwallet->AddKeyPubKey(key, pubkey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); - } - pwallet->LearnAllRelatedScripts(pubkey); + CKey key = vchSecret.GetKey(); + if (!key.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Private key outside allowed range"); - // whenever a key is imported, we need to scan the whole chain - pwallet->UpdateTimeFirstKey(1); + CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); + CKeyID vchAddress = pubkey.GetID(); + { + pwallet->MarkDirty(); + // We don't know which corresponding address will be used; label them all + for (const auto& dest : GetAllDestinationsForKey(pubkey)) { + pwallet->SetAddressBook(dest, strLabel, "receive"); + } - if (fRescan) { - pwallet->RescanFromTime(TIMESTAMP_MIN, true /* update */); + // Don't throw error in case a key is already there + if (pwallet->HaveKey(vchAddress)) { + return NullUniValue; + } + + // whenever a key is imported, we need to scan the whole chain + pwallet->UpdateTimeFirstKey(1); + pwallet->mapKeyMetadata[vchAddress].nCreateTime = 1; + + if (!pwallet->AddKeyPubKey(key, pubkey)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + } + pwallet->LearnAllRelatedScripts(pubkey); } } + if (fRescan) { + pwallet->RescanFromTime(TIMESTAMP_MIN, reserver, true /* update */); + } return NullUniValue; } @@ -237,7 +242,8 @@ UniValue importaddress(const JSONRPCRequest& request) "2. \"label\" (string, optional, default=\"\") An optional label\n" "3. rescan (boolean, optional, default=true) Rescan the wallet for transactions\n" "4. p2sh (boolean, optional, default=false) Add the P2SH version of the script as well\n" - "\nNote: This call can take minutes to complete if rescan is true.\n" + "\nNote: This call can take minutes to complete if rescan is true, during that time, other rpc calls\n" + "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" "If you have the full public key, you should call importpubkey instead of this.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" "as change, and not show up in many RPCs.\n" @@ -263,29 +269,35 @@ UniValue importaddress(const JSONRPCRequest& request) if (fRescan && fPruneMode) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + WalletRescanReserver reserver(pwallet); + if (fRescan && !reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } + // Whether to import a p2sh version, too bool fP2SH = false; if (!request.params[3].isNull()) fP2SH = request.params[3].get_bool(); - LOCK2(cs_main, pwallet->cs_wallet); + { + LOCK2(cs_main, pwallet->cs_wallet); - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (IsValidDestination(dest)) { - if (fP2SH) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Cannot use the p2sh flag with an address - use a script instead"); + CTxDestination dest = DecodeDestination(request.params[0].get_str()); + if (IsValidDestination(dest)) { + if (fP2SH) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Cannot use the p2sh flag with an address - use a script instead"); + } + ImportAddress(pwallet, dest, strLabel); + } else if (IsHex(request.params[0].get_str())) { + std::vector<unsigned char> data(ParseHex(request.params[0].get_str())); + ImportScript(pwallet, CScript(data.begin(), data.end()), strLabel, fP2SH); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address or script"); } - ImportAddress(pwallet, dest, strLabel); - } else if (IsHex(request.params[0].get_str())) { - std::vector<unsigned char> data(ParseHex(request.params[0].get_str())); - ImportScript(pwallet, CScript(data.begin(), data.end()), strLabel, fP2SH); - } else { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address or script"); } - if (fRescan) { - pwallet->RescanFromTime(TIMESTAMP_MIN, true /* update */); + pwallet->RescanFromTime(TIMESTAMP_MIN, reserver, true /* update */); pwallet->ReacceptWalletTransactions(); } @@ -406,7 +418,8 @@ UniValue importpubkey(const JSONRPCRequest& request) "1. \"pubkey\" (string, required) The hex-encoded public key\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" "3. rescan (boolean, optional, default=true) Rescan the wallet for transactions\n" - "\nNote: This call can take minutes to complete if rescan is true.\n" + "\nNote: This call can take minutes to complete if rescan is true, during that time, other rpc calls\n" + "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" "\nExamples:\n" "\nImport a public key with rescan\n" + HelpExampleCli("importpubkey", "\"mypubkey\"") + @@ -429,6 +442,11 @@ UniValue importpubkey(const JSONRPCRequest& request) if (fRescan && fPruneMode) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + WalletRescanReserver reserver(pwallet); + if (fRescan && !reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } + if (!IsHex(request.params[0].get_str())) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey must be a hex string"); std::vector<unsigned char> data(ParseHex(request.params[0].get_str())); @@ -436,17 +454,18 @@ UniValue importpubkey(const JSONRPCRequest& request) if (!pubKey.IsFullyValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); - LOCK2(cs_main, pwallet->cs_wallet); + { + LOCK2(cs_main, pwallet->cs_wallet); - for (const auto& dest : GetAllDestinationsForKey(pubKey)) { - ImportAddress(pwallet, dest, strLabel); + for (const auto& dest : GetAllDestinationsForKey(pubKey)) { + ImportAddress(pwallet, dest, strLabel); + } + ImportScript(pwallet, GetScriptForRawPubKey(pubKey), strLabel, false); + pwallet->LearnAllRelatedScripts(pubKey); } - ImportScript(pwallet, GetScriptForRawPubKey(pubKey), strLabel, false); - pwallet->LearnAllRelatedScripts(pubKey); - if (fRescan) { - pwallet->RescanFromTime(TIMESTAMP_MIN, true /* update */); + pwallet->RescanFromTime(TIMESTAMP_MIN, reserver, true /* update */); pwallet->ReacceptWalletTransactions(); } @@ -479,91 +498,98 @@ UniValue importwallet(const JSONRPCRequest& request) if (fPruneMode) throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled in pruned mode"); - LOCK2(cs_main, pwallet->cs_wallet); + WalletRescanReserver reserver(pwallet); + if (!reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } - EnsureWalletIsUnlocked(pwallet); + int64_t nTimeBegin = 0; + bool fGood = true; + { + LOCK2(cs_main, pwallet->cs_wallet); - std::ifstream file; - file.open(request.params[0].get_str().c_str(), std::ios::in | std::ios::ate); - if (!file.is_open()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); + EnsureWalletIsUnlocked(pwallet); - int64_t nTimeBegin = chainActive.Tip()->GetBlockTime(); + std::ifstream file; + file.open(request.params[0].get_str().c_str(), std::ios::in | std::ios::ate); + if (!file.is_open()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); + } + nTimeBegin = chainActive.Tip()->GetBlockTime(); - bool fGood = true; + int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); + file.seekg(0, file.beg); - int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); - file.seekg(0, file.beg); - - pwallet->ShowProgress(_("Importing..."), 0); // show progress dialog in GUI - while (file.good()) { - pwallet->ShowProgress("", std::max(1, std::min(99, (int)(((double)file.tellg() / (double)nFilesize) * 100)))); - std::string line; - std::getline(file, line); - if (line.empty() || line[0] == '#') - continue; - - std::vector<std::string> vstr; - boost::split(vstr, line, boost::is_any_of(" ")); - if (vstr.size() < 2) - continue; - CBitcoinSecret vchSecret; - if (vchSecret.SetString(vstr[0])) { - CKey key = vchSecret.GetKey(); - CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); - CKeyID keyid = pubkey.GetID(); - if (pwallet->HaveKey(keyid)) { - LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid)); + pwallet->ShowProgress(_("Importing..."), 0); // show progress dialog in GUI + while (file.good()) { + pwallet->ShowProgress("", std::max(1, std::min(99, (int)(((double)file.tellg() / (double)nFilesize) * 100)))); + std::string line; + std::getline(file, line); + if (line.empty() || line[0] == '#') continue; - } - int64_t nTime = DecodeDumpTime(vstr[1]); - std::string strLabel; - bool fLabel = true; - for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { - if (boost::algorithm::starts_with(vstr[nStr], "#")) - break; - if (vstr[nStr] == "change=1") - fLabel = false; - if (vstr[nStr] == "reserve=1") - fLabel = false; - if (boost::algorithm::starts_with(vstr[nStr], "label=")) { - strLabel = DecodeDumpString(vstr[nStr].substr(6)); - fLabel = true; - } - } - LogPrintf("Importing %s...\n", EncodeDestination(keyid)); - if (!pwallet->AddKeyPubKey(key, pubkey)) { - fGood = false; + + std::vector<std::string> vstr; + boost::split(vstr, line, boost::is_any_of(" ")); + if (vstr.size() < 2) continue; + CBitcoinSecret vchSecret; + if (vchSecret.SetString(vstr[0])) { + CKey key = vchSecret.GetKey(); + CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); + CKeyID keyid = pubkey.GetID(); + if (pwallet->HaveKey(keyid)) { + LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid)); + continue; + } + int64_t nTime = DecodeDumpTime(vstr[1]); + std::string strLabel; + bool fLabel = true; + for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { + if (boost::algorithm::starts_with(vstr[nStr], "#")) + break; + if (vstr[nStr] == "change=1") + fLabel = false; + if (vstr[nStr] == "reserve=1") + fLabel = false; + if (boost::algorithm::starts_with(vstr[nStr], "label=")) { + strLabel = DecodeDumpString(vstr[nStr].substr(6)); + fLabel = true; + } + } + LogPrintf("Importing %s...\n", EncodeDestination(keyid)); + if (!pwallet->AddKeyPubKey(key, pubkey)) { + fGood = false; + continue; + } + pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; + if (fLabel) + pwallet->SetAddressBook(keyid, strLabel, "receive"); + nTimeBegin = std::min(nTimeBegin, nTime); + } else if(IsHex(vstr[0])) { + std::vector<unsigned char> vData(ParseHex(vstr[0])); + CScript script = CScript(vData.begin(), vData.end()); + if (pwallet->HaveCScript(script)) { + LogPrintf("Skipping import of %s (script already present)\n", vstr[0]); + continue; + } + if(!pwallet->AddCScript(script)) { + LogPrintf("Error importing script %s\n", vstr[0]); + fGood = false; + continue; + } + int64_t birth_time = DecodeDumpTime(vstr[1]); + if (birth_time > 0) { + pwallet->m_script_metadata[CScriptID(script)].nCreateTime = birth_time; + nTimeBegin = std::min(nTimeBegin, birth_time); + } } - pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; - if (fLabel) - pwallet->SetAddressBook(keyid, strLabel, "receive"); - nTimeBegin = std::min(nTimeBegin, nTime); - } else if(IsHex(vstr[0])) { - std::vector<unsigned char> vData(ParseHex(vstr[0])); - CScript script = CScript(vData.begin(), vData.end()); - if (pwallet->HaveCScript(script)) { - LogPrintf("Skipping import of %s (script already present)\n", vstr[0]); - continue; - } - if(!pwallet->AddCScript(script)) { - LogPrintf("Error importing script %s\n", vstr[0]); - fGood = false; - continue; - } - int64_t birth_time = DecodeDumpTime(vstr[1]); - if (birth_time > 0) { - pwallet->m_script_metadata[CScriptID(script)].nCreateTime = birth_time; - nTimeBegin = std::min(nTimeBegin, birth_time); - } } + file.close(); + pwallet->ShowProgress("", 100); // hide progress dialog in GUI + pwallet->UpdateTimeFirstKey(nTimeBegin); } - file.close(); - pwallet->ShowProgress("", 100); // hide progress dialog in GUI - pwallet->UpdateTimeFirstKey(nTimeBegin); - pwallet->RescanFromTime(nTimeBegin, false /* update */); + pwallet->RescanFromTime(nTimeBegin, reserver, false /* update */); pwallet->MarkDirty(); if (!fGood) @@ -685,7 +711,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) file << strprintf("# mined on %s\n", EncodeDumpTime(chainActive.Tip()->GetBlockTime())); file << "\n"; - // add the base58check encoded extended master if the wallet uses HD + // add the base58check encoded extended master if the wallet uses HD CKeyID masterKeyID = pwallet->GetHDChain().masterKeyID; if (!masterKeyID.IsNull()) { @@ -1110,6 +1136,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) " {\n" " \"rescan\": <false>, (boolean, optional, default: true) Stating if should rescan the blockchain after all imports\n" " }\n" + "\nNote: This call can take minutes to complete if rescan is true, during that time, other rpc calls\n" + "may report that the imported keys, addresses or scripts exists but related transactions are still missing.\n" "\nExamples:\n" + HelpExampleCli("importmulti", "'[{ \"scriptPubKey\": { \"address\": \"<my address>\" }, \"timestamp\":1455191478 }, " "{ \"scriptPubKey\": { \"address\": \"<my 2nd address>\" }, \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + @@ -1135,49 +1163,55 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } } - LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(pwallet); - - // Verify all timestamps are present before importing any keys. - const int64_t now = chainActive.Tip() ? chainActive.Tip()->GetMedianTimePast() : 0; - for (const UniValue& data : requests.getValues()) { - GetImportTimestamp(data, now); + WalletRescanReserver reserver(pwallet); + if (fRescan && !reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } + int64_t now = 0; bool fRunScan = false; - const int64_t minimumTimestamp = 1; int64_t nLowestTimestamp = 0; - - if (fRescan && chainActive.Tip()) { - nLowestTimestamp = chainActive.Tip()->GetBlockTime(); - } else { - fRescan = false; - } - UniValue response(UniValue::VARR); + { + LOCK2(cs_main, pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); - for (const UniValue& data : requests.getValues()) { - const int64_t timestamp = std::max(GetImportTimestamp(data, now), minimumTimestamp); - const UniValue result = ProcessImport(pwallet, data, timestamp); - response.push_back(result); - - if (!fRescan) { - continue; + // Verify all timestamps are present before importing any keys. + now = chainActive.Tip() ? chainActive.Tip()->GetMedianTimePast() : 0; + for (const UniValue& data : requests.getValues()) { + GetImportTimestamp(data, now); } - // If at least one request was successful then allow rescan. - if (result["success"].get_bool()) { - fRunScan = true; + const int64_t minimumTimestamp = 1; + + if (fRescan && chainActive.Tip()) { + nLowestTimestamp = chainActive.Tip()->GetBlockTime(); + } else { + fRescan = false; } - // Get the lowest timestamp. - if (timestamp < nLowestTimestamp) { - nLowestTimestamp = timestamp; + for (const UniValue& data : requests.getValues()) { + const int64_t timestamp = std::max(GetImportTimestamp(data, now), minimumTimestamp); + const UniValue result = ProcessImport(pwallet, data, timestamp); + response.push_back(result); + + if (!fRescan) { + continue; + } + + // If at least one request was successful then allow rescan. + if (result["success"].get_bool()) { + fRunScan = true; + } + + // Get the lowest timestamp. + if (timestamp < nLowestTimestamp) { + nLowestTimestamp = timestamp; + } } } - if (fRescan && fRunScan && requests.size()) { - int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, true /* update */); + int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); pwallet->ReacceptWalletTransactions(); if (scannedTime > nLowestTimestamp) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index bd8c9ecb0c..a77a886428 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -145,7 +145,7 @@ UniValue getnewaddress(const JSONRPCRequest& request) "so payments received with the address will be credited to 'account'.\n" "\nArguments:\n" "1. \"account\" (string, optional) DEPRECATED. The account name for the address to be linked to. If not provided, the default account \"\" is used. It can also be set to the empty string \"\" to represent the default account. The account does not need to exist, it will be created if there is no account by the given name.\n" - "2. \"address_type\" (string, optional) The address type to use. Options are \"legacy\", \"p2sh\", and \"bech32\". Default is set by -addresstype.\n" + "2. \"address_type\" (string, optional) The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\". Default is set by -addresstype.\n" "\nResult:\n" "\"address\" (string) The new bitcoin address\n" "\nExamples:\n" @@ -243,7 +243,7 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) "\nReturns a new Bitcoin address, for receiving change.\n" "This is for use with raw transactions, NOT normal use.\n" "\nArguments:\n" - "1. \"address_type\" (string, optional) The address type to use. Options are \"legacy\", \"p2sh\", and \"bech32\". Default is set by -changetype.\n" + "1. \"address_type\" (string, optional) The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\". Default is set by -changetype.\n" "\nResult:\n" "\"address\" (string) The address\n" "\nExamples:\n" @@ -1309,7 +1309,8 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { std::string msg = "addwitnessaddress \"address\" ( p2sh )\n" - "\nAdd a witness address for a script (with pubkey or redeemscript known). Requires a new wallet backup.\n" + "\nDEPRECATED: set the address_type argument of getnewaddress, or option -addresstype=[bech32|p2sh-segwit] instead.\n" + "Add a witness address for a script (with pubkey or redeemscript known). Requires a new wallet backup.\n" "It returns the witness script.\n" "\nArguments:\n" @@ -1323,6 +1324,12 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) throw std::runtime_error(msg); } + if (!IsDeprecatedRPCEnabled("addwitnessaddress")) { + throw JSONRPCError(RPC_METHOD_DEPRECATED, "addwitnessaddress is deprecated and will be fully removed in v0.17. " + "To use addwitnessaddress in v0.16, restart bitcoind with -deprecatedrpc=addwitnessaddress.\n" + "Projects should transition to using the address_type argument of getnewaddress, or option -addresstype=[bech32|p2sh-segwit] instead.\n"); + } + { LOCK(cs_main); if (!IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()) && !gArgs.GetBoolArg("-walletprematurewitness", false)) { @@ -2307,7 +2314,8 @@ UniValue walletpassphrase(const JSONRPCRequest& request) "This is needed prior to performing transactions related to private keys such as sending bitcoins\n" "\nArguments:\n" "1. \"passphrase\" (string, required) The wallet passphrase\n" - "2. timeout (numeric, required) The time to keep the decryption key in seconds.\n" + "2. timeout (numeric, required) The time to keep the decryption key in seconds. Limited to at most 1073741824 (2^30) seconds.\n" + " Any value greater than 1073741824 seconds will be set to 1073741824 seconds.\n" "\nNote:\n" "Issuing the walletpassphrase command while the wallet is already unlocked will set a new unlock\n" "time that overrides the old one.\n" @@ -2336,6 +2344,17 @@ UniValue walletpassphrase(const JSONRPCRequest& request) // Alternately, find a way to make request.params[0] mlock()'d to begin with. strWalletPass = request.params[0].get_str().c_str(); + // Get the timeout + int64_t nSleepTime = request.params[1].get_int64(); + // Timeout cannot be negative, otherwise it will relock immediately + if (nSleepTime < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Timeout cannot be negative."); + } + // Clamp timeout to 2^30 seconds + if (nSleepTime > (int64_t)1 << 30) { + nSleepTime = (int64_t)1 << 30; + } + if (strWalletPass.length() > 0) { if (!pwallet->Unlock(strWalletPass)) { @@ -2349,7 +2368,6 @@ UniValue walletpassphrase(const JSONRPCRequest& request) pwallet->TopUpKeyPool(); - int64_t nSleepTime = request.params[1].get_int64(); pwallet->nRelockTime = GetTime() + nSleepTime; RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), boost::bind(LockWallet, pwallet), nSleepTime); @@ -3422,30 +3440,41 @@ UniValue rescanblockchain(const JSONRPCRequest& request) ); } - LOCK2(cs_main, pwallet->cs_wallet); + WalletRescanReserver reserver(pwallet); + if (!reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } - CBlockIndex *pindexStart = chainActive.Genesis(); + CBlockIndex *pindexStart = nullptr; CBlockIndex *pindexStop = nullptr; - if (!request.params[0].isNull()) { - pindexStart = chainActive[request.params[0].get_int()]; - if (!pindexStart) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); - } - } + CBlockIndex *pChainTip = nullptr; + { + LOCK(cs_main); + pindexStart = chainActive.Genesis(); + pChainTip = chainActive.Tip(); - if (!request.params[1].isNull()) { - pindexStop = chainActive[request.params[1].get_int()]; - if (!pindexStop) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); + if (!request.params[0].isNull()) { + pindexStart = chainActive[request.params[0].get_int()]; + if (!pindexStart) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); + } } - else if (pindexStop->nHeight < pindexStart->nHeight) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "stop_height must be greater then start_height"); + + if (!request.params[1].isNull()) { + pindexStop = chainActive[request.params[1].get_int()]; + if (!pindexStop) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); + } + else if (pindexStop->nHeight < pindexStart->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "stop_height must be greater then start_height"); + } } } // We can't rescan beyond non-pruned blocks, stop and throw an error if (fPruneMode) { - CBlockIndex *block = pindexStop ? pindexStop : chainActive.Tip(); + LOCK(cs_main); + CBlockIndex *block = pindexStop ? pindexStop : pChainTip; while (block && block->nHeight >= pindexStart->nHeight) { if (!(block->nStatus & BLOCK_HAVE_DATA)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height."); @@ -3454,18 +3483,17 @@ UniValue rescanblockchain(const JSONRPCRequest& request) } } - CBlockIndex *stopBlock = pwallet->ScanForWalletTransactions(pindexStart, pindexStop, true); + CBlockIndex *stopBlock = pwallet->ScanForWalletTransactions(pindexStart, pindexStop, reserver, true); if (!stopBlock) { if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted."); } // if we got a nullptr returned, ScanForWalletTransactions did rescan up to the requested stopindex - stopBlock = pindexStop ? pindexStop : chainActive.Tip(); + stopBlock = pindexStop ? pindexStop : pChainTip; } else { throw JSONRPCError(RPC_MISC_ERROR, "Rescan failed. Potentially corrupted data files."); } - UniValue response(UniValue::VOBJ); response.pushKV("start_height", pindexStart->nHeight); response.pushKV("stop_height", stopBlock->nHeight); @@ -3492,7 +3520,7 @@ static const CRPCCommand commands[] = { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, { "wallet", "abortrescan", &abortrescan, {} }, { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","account"} }, - { "wallet", "addwitnessaddress", &addwitnessaddress, {"address","p2sh"} }, + { "hidden", "addwitnessaddress", &addwitnessaddress, {"address","p2sh"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 7b3c283f37..7e0881afd7 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -384,7 +384,9 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) { CWallet wallet; AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip, nullptr)); + WalletRescanReserver reserver(&wallet); + reserver.reserve(); + BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip, nullptr, reserver)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); } @@ -397,7 +399,9 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) { CWallet wallet; AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip, nullptr)); + WalletRescanReserver reserver(&wallet); + reserver.reserve(); + BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip, nullptr, reserver)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); } @@ -608,7 +612,9 @@ public: bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - wallet->ScanForWalletTransactions(chainActive.Genesis(), nullptr); + WalletRescanReserver reserver(wallet.get()); + reserver.reserve(); + wallet->ScanForWalletTransactions(chainActive.Genesis(), nullptr, reserver); } ~ListCoinsTestingSetup() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index da721ea008..dd9bc64728 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1612,19 +1612,20 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, * @return Earliest timestamp that could be successfully scanned from. Timestamp * returned will be higher than startTime if relevant blocks could not be read. */ -int64_t CWallet::RescanFromTime(int64_t startTime, bool update) +int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update) { - AssertLockHeld(cs_main); - AssertLockHeld(cs_wallet); - // Find starting block. May be null if nCreateTime is greater than the // highest blockchain timestamp, in which case there is nothing that needs // to be scanned. - CBlockIndex* const startBlock = chainActive.FindEarliestAtLeast(startTime - TIMESTAMP_WINDOW); - LogPrintf("%s: Rescanning last %i blocks\n", __func__, startBlock ? chainActive.Height() - startBlock->nHeight + 1 : 0); + CBlockIndex* startBlock = nullptr; + { + LOCK(cs_main); + startBlock = chainActive.FindEarliestAtLeast(startTime - TIMESTAMP_WINDOW); + LogPrintf("%s: Rescanning last %i blocks\n", __func__, startBlock ? chainActive.Height() - startBlock->nHeight + 1 : 0); + } if (startBlock) { - const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, nullptr, update); + const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, nullptr, reserver, update); if (failedBlock) { return failedBlock->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1; } @@ -1643,12 +1644,17 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update) * * If pindexStop is not a nullptr, the scan will stop at the block-index * defined by pindexStop + * + * Caller needs to make sure pindexStop (and the optional pindexStart) are on + * the main chain after to the addition of any new keys you want to detect + * transactions for. */ -CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate) +CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, const WalletRescanReserver &reserver, bool fUpdate) { int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); + assert(reserver.isReserved()); if (pindexStop) { assert(pindexStop->nHeight >= pindexStart->nHeight); } @@ -1656,24 +1662,42 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlock CBlockIndex* pindex = pindexStart; CBlockIndex* ret = nullptr; { - LOCK2(cs_main, cs_wallet); fAbortRescan = false; - fScanningWallet = true; - ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); - double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); + CBlockIndex* tip = nullptr; + double dProgressStart; + double dProgressTip; + { + LOCK(cs_main); + tip = chainActive.Tip(); + dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); + dProgressTip = GuessVerificationProgress(chainParams.TxData(), tip); + } while (pindex && !fAbortRescan) { - if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) - ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) { + double gvp = 0; + { + LOCK(cs_main); + gvp = GuessVerificationProgress(chainParams.TxData(), pindex); + } + ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((gvp - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + } if (GetTime() >= nNow + 60) { nNow = GetTime(); + LOCK(cs_main); LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { + LOCK2(cs_main, cs_wallet); + if (pindex && !chainActive.Contains(pindex)) { + // Abort scan if current block is no longer active, to prevent + // marking transactions as coming from the wrong block. + ret = pindex; + break; + } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate); } @@ -1683,14 +1707,20 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlock if (pindex == pindexStop) { break; } - pindex = chainActive.Next(pindex); + { + LOCK(cs_main); + pindex = chainActive.Next(pindex); + if (tip != chainActive.Tip()) { + tip = chainActive.Tip(); + // in case the tip has changed, update progress max + dProgressTip = GuessVerificationProgress(chainParams.TxData(), tip); + } + } } if (pindex && fAbortRescan) { LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI - - fScanningWallet = false; } return ret; } @@ -4001,7 +4031,14 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) } nStart = GetTimeMillis(); - walletInstance->ScanForWalletTransactions(pindexRescan, nullptr, true); + { + WalletRescanReserver reserver(walletInstance); + if (!reserver.reserve()) { + InitError(_("Failed to rescan the wallet during initialization")); + return nullptr; + } + walletInstance->ScanForWalletTransactions(pindexRescan, nullptr, reserver, true); + } LogPrintf(" rescan %15dms\n", GetTimeMillis() - nStart); walletInstance->SetBestChain(chainActive.GetLocator()); walletInstance->dbw->IncrementUpdateCounter(); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 53a2c6b9d5..354f524821 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -99,7 +99,7 @@ enum WalletFeature FEATURE_LATEST = FEATURE_COMPRPUBKEY // HD is optional, use FEATURE_COMPRPUBKEY as latest version }; -enum OutputType +enum OutputType : int { OUTPUT_TYPE_NONE, OUTPUT_TYPE_LEGACY, @@ -659,6 +659,7 @@ private: }; +class WalletRescanReserver; //forward declarations for ScanForWalletTransactions/RescanFromTime /** * A CWallet is an extension of a keystore, which also maintains a set of transactions and balances, * and provides the ability to create new transactions. @@ -668,7 +669,10 @@ class CWallet final : public CCryptoKeyStore, public CValidationInterface private: static std::atomic<bool> fFlushScheduled; std::atomic<bool> fAbortRescan; - std::atomic<bool> fScanningWallet; + std::atomic<bool> fScanningWallet; //controlled by WalletRescanReserver + std::mutex mutexScanning; + friend class WalletRescanReserver; + /** * Select a set of coins such that nValueRet >= nTargetValue and at least @@ -945,8 +949,8 @@ public: void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) override; void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); - int64_t RescanFromTime(int64_t startTime, bool update); - CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate = false); + int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); + CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, const WalletRescanReserver& reserver, bool fUpdate = false); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; @@ -1263,4 +1267,39 @@ CTxDestination GetDestinationForKey(const CPubKey& key, OutputType); /** Get all destinations (potentially) supported by the wallet for the given key. */ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); +/** RAII object to check and reserve a wallet rescan */ +class WalletRescanReserver +{ +private: + CWalletRef m_wallet; + bool m_could_reserve; +public: + explicit WalletRescanReserver(CWalletRef w) : m_wallet(w), m_could_reserve(false) {} + + bool reserve() + { + assert(!m_could_reserve); + std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); + if (m_wallet->fScanningWallet) { + return false; + } + m_wallet->fScanningWallet = true; + m_could_reserve = true; + return true; + } + + bool isReserved() const + { + return (m_could_reserve && m_wallet->fScanningWallet); + } + + ~WalletRescanReserver() + { + std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); + if (m_could_reserve) { + m_wallet->fScanningWallet = false; + } + } +}; + #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index f15e5de1e2..7c97b668ae 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -9,7 +9,7 @@ fs::path GetWalletDir() fs::path path; if (gArgs.IsArgSet("-walletdir")) { - path = fs::system_complete(gArgs.GetArg("-walletdir", "")); + path = gArgs.GetArg("-walletdir", ""); if (!fs::is_directory(path)) { // If the path specified doesn't exist, we return the deliberately // invalid empty string. diff --git a/test/README.md b/test/README.md index 868eb667ae..b59c8db4e5 100644 --- a/test/README.md +++ b/test/README.md @@ -8,7 +8,7 @@ There are currently two sets of tests in this directory: - [functional](/test/functional) which test the functionality of bitcoind and bitcoin-qt by interacting with them through the RPC and P2P interfaces. -- [util](test/util) which tests the bitcoin utilities, currently only +- [util](/test/util) which tests the bitcoin utilities, currently only bitcoin-tx. The util tests are run as part of `make check` target. The functional diff --git a/test/functional/README.md b/test/functional/README.md index 4d52751b05..d6ce490ab3 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -27,6 +27,20 @@ don't have test cases for. `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. +#### Naming guidelines + +- Name the test `<area>_test.py`, where area can be one of the following: + - `feature` for tests for full features that aren't wallet/mining/mempool, eg `feature_rbf.py` + - `interface` for tests for other interfaces (REST, ZMQ, etc), eg `interface_rest.py` + - `mempool` for tests for mempool behaviour, eg `mempool_reorg.py` + - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` + - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` + - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` + - `wallet` for tests for wallet features, eg `wallet_keypool.py` +- use an underscore to separate words + - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` +- Don't use the redundant word `test` in the name, eg `interface_zmq.py`, not `interface_zmq_test.py` + #### General test-writing advice - Set `self.num_nodes` to the minimum number of nodes necessary for the test. @@ -73,7 +87,7 @@ start the networking thread. (Continue with the test logic in your existing thread.) - Can be used to write tests where specific P2P protocol behavior is tested. -Examples tests are `p2p-accept-block.py`, `p2p-compactblocks.py`. +Examples tests are `p2p-acceptblock.py`, `p2p-compactblocks.py`. #### Comptool diff --git a/test/functional/bumpfee.py b/test/functional/bumpfee.py index 5cbd9f5cf7..2cd4127854 100755 --- a/test/functional/bumpfee.py +++ b/test/functional/bumpfee.py @@ -14,7 +14,7 @@ added in the future, they should try to follow the same convention and not make assumptions about execution order. """ -from segwit import send_to_witness +from test_framework.blocktools import send_to_witness from test_framework.test_framework import BitcoinTestFramework from test_framework import blocktools from test_framework.mininode import CTransaction @@ -33,7 +33,7 @@ class BumpFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [["-prematurewitness", "-walletprematurewitness", "-walletrbf={}".format(i)] + self.extra_args = [["-prematurewitness", "-walletprematurewitness", "-deprecatedrpc=addwitnessaddress", "-walletrbf={}".format(i)] for i in range(self.num_nodes)] def run_test(self): diff --git a/test/functional/create_cache.py b/test/functional/create_cache.py index 4c79814a26..9665c50a92 100755 --- a/test/functional/create_cache.py +++ b/test/functional/create_cache.py @@ -16,6 +16,7 @@ class CreateCache(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 0 + self.supports_cli = True def setup_network(self): pass diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index d0c40e5446..b07e451667 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -15,63 +15,75 @@ from test_framework.util import assert_equal, assert_raises_rpc_error class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3', '-wallet=w']] + self.num_nodes = 2 + self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3', '-wallet=w'], []] + self.supports_cli = True def run_test(self): - assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3", "w"}) + node = self.nodes[0] - self.stop_node(0) + data_dir = lambda *p: os.path.join(node.datadir, 'regtest', *p) + wallet_dir = lambda *p: data_dir('wallets', *p) + wallet = lambda name: node.get_wallet_rpc(name) + + assert_equal(set(node.listwallets()), {"w1", "w2", "w3", "w"}) + + self.stop_nodes() + + self.assert_start_raises_init_error(0, ['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist') + self.assert_start_raises_init_error(0, ['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" is a relative path', cwd=data_dir()) + self.assert_start_raises_init_error(0, ['-walletdir=debug.log'], 'Error: Specified -walletdir "debug.log" is not a directory', cwd=data_dir()) # should not initialize if there are duplicate wallets self.assert_start_raises_init_error(0, ['-wallet=w1', '-wallet=w1'], 'Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if wallet file is a directory - wallet_dir = os.path.join(self.options.tmpdir, 'node0', 'regtest', 'wallets') - os.mkdir(os.path.join(wallet_dir, 'w11')) + os.mkdir(wallet_dir('w11')) self.assert_start_raises_init_error(0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') # should not initialize if one wallet is a copy of another - shutil.copyfile(os.path.join(wallet_dir, 'w2'), os.path.join(wallet_dir, 'w22')) + shutil.copyfile(wallet_dir('w2'), wallet_dir('w22')) self.assert_start_raises_init_error(0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') # should not initialize if wallet file is a symlink - os.symlink(os.path.join(wallet_dir, 'w1'), os.path.join(wallet_dir, 'w12')) + os.symlink(wallet_dir('w1'), wallet_dir('w12')) self.assert_start_raises_init_error(0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') # should not initialize if the specified walletdir does not exist self.assert_start_raises_init_error(0, ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory - not_a_dir = os.path.join(wallet_dir, 'notadir') + not_a_dir = wallet_dir('notadir') open(not_a_dir, 'a').close() - self.assert_start_raises_init_error(0, ['-walletdir='+not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') + self.assert_start_raises_init_error(0, ['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') # if wallets/ doesn't exist, datadir should be the default wallet dir - wallet_dir2 = os.path.join(self.options.tmpdir, 'node0', 'regtest', 'walletdir') - os.rename(wallet_dir, wallet_dir2) + wallet_dir2 = data_dir('walletdir') + os.rename(wallet_dir(), wallet_dir2) self.start_node(0, ['-wallet=w4', '-wallet=w5']) - assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) - w5 = self.nodes[0].get_wallet_rpc("w5") + assert_equal(set(node.listwallets()), {"w4", "w5"}) + w5 = wallet("w5") w5.generate(1) - self.stop_node(0) # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded - os.rename(wallet_dir2, wallet_dir) - self.start_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + os.path.join(self.options.tmpdir, 'node0', 'regtest')]) - assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) - w5 = self.nodes[0].get_wallet_rpc("w5") + os.rename(wallet_dir2, wallet_dir()) + self.restart_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + data_dir()]) + assert_equal(set(node.listwallets()), {"w4", "w5"}) + w5 = wallet("w5") w5_info = w5.getwalletinfo() assert_equal(w5_info['immature_balance'], 50) - self.stop_node(0) + competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir') + os.mkdir(competing_wallet_dir) + self.restart_node(0, ['-walletdir='+competing_wallet_dir]) + self.assert_start_raises_init_error(1, ['-walletdir='+competing_wallet_dir], 'Error initializing wallet database environment') - self.start_node(0, self.extra_args[0]) + self.restart_node(0, self.extra_args[0]) - w1 = self.nodes[0].get_wallet_rpc("w1") - w2 = self.nodes[0].get_wallet_rpc("w2") - w3 = self.nodes[0].get_wallet_rpc("w3") - w4 = self.nodes[0].get_wallet_rpc("w") - wallet_bad = self.nodes[0].get_wallet_rpc("bad") + w1 = wallet("w1") + w2 = wallet("w2") + w3 = wallet("w3") + w4 = wallet("w") + wallet_bad = wallet("bad") w1.generate(1) @@ -79,7 +91,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails - assert_raises_rpc_error(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) + assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) # check w1 wallet balance w1_info = w1.getwalletinfo() diff --git a/test/functional/nulldummy.py b/test/functional/nulldummy.py index 1fbf4c1381..fac620bc8d 100755 --- a/test/functional/nulldummy.py +++ b/test/functional/nulldummy.py @@ -42,7 +42,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.setup_clean_chain = True # This script tests NULLDUMMY activation, which is part of the 'segwit' deployment, so we go through # normal segwit activation here (and don't use the default always-on behaviour). - self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness', '-vbparams=segwit:0:999999999999', '-addresstype=legacy']] + self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness', '-vbparams=segwit:0:999999999999', '-addresstype=legacy', "-deprecatedrpc=addwitnessaddress"]] def run_test(self): self.address = self.nodes[0].getnewaddress() diff --git a/test/functional/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py index e98ae31a89..d9f461a049 100755 --- a/test/functional/p2p-compactblocks.py +++ b/test/functional/p2p-compactblocks.py @@ -95,7 +95,7 @@ class CompactBlocksTest(BitcoinTestFramework): self.num_nodes = 2 # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). # TODO: Rewrite this test to support SegWit being always active. - self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] + self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex", "-deprecatedrpc=addwitnessaddress"]] self.utxos = [] def build_block_on_tip(self, node, segwit=False): diff --git a/test/functional/segwit.py b/test/functional/segwit.py index d3de23e061..931ebdea39 100755 --- a/test/functional/segwit.py +++ b/test/functional/segwit.py @@ -4,10 +4,18 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" +from test_framework.address import ( + key_to_p2sh_p2wpkh, + key_to_p2wpkh, + program_to_witness, + script_to_p2sh_p2wsh, + script_to_p2wsh, +) +from test_framework.blocktools import witness_script, send_to_witness from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.mininode import sha256, CTransaction, CTxIn, COutPoint, CTxOut, COIN, ToHex, FromHex -from test_framework.address import script_to_p2sh, key_to_p2pkh, key_to_p2sh_p2wpkh, key_to_p2wpkh, script_to_p2sh_p2wsh, script_to_p2wsh, program_to_witness +from test_framework.address import script_to_p2sh, key_to_p2pkh from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG, OP_TRUE from io import BytesIO @@ -16,52 +24,6 @@ NODE_2 = 2 WIT_V0 = 0 WIT_V1 = 1 -# Create a scriptPubKey corresponding to either a P2WPKH output for the -# given pubkey, or a P2WSH output of a 1-of-1 multisig for the given -# pubkey. Returns the hex encoding of the scriptPubKey. -def witness_script(use_p2wsh, pubkey): - if (use_p2wsh == False): - # P2WPKH instead - pubkeyhash = hash160(hex_str_to_bytes(pubkey)) - pkscript = CScript([OP_0, pubkeyhash]) - else: - # 1-of-1 multisig - witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) - scripthash = sha256(witness_program) - pkscript = CScript([OP_0, scripthash]) - return bytes_to_hex_str(pkscript) - -# Return a transaction (in hex) that spends the given utxo to a segwit output, -# optionally wrapping the segwit output using P2SH. -def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): - if use_p2wsh: - program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) - addr = script_to_p2sh_p2wsh(program) if encode_p2sh else script_to_p2wsh(program) - else: - addr = key_to_p2sh_p2wpkh(pubkey) if encode_p2sh else key_to_p2wpkh(pubkey) - if not encode_p2sh: - assert_equal(node.validateaddress(addr)['scriptPubKey'], witness_script(use_p2wsh, pubkey)) - return node.createrawtransaction([utxo], {addr: amount}) - -# Create a transaction spending a given utxo to a segwit output corresponding -# to the given pubkey: use_p2wsh determines whether to use P2WPKH or P2WSH; -# encode_p2sh determines whether to wrap in P2SH. -# sign=True will have the given node sign the transaction. -# insert_redeem_script will be added to the scriptSig, if given. -def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=True, insert_redeem_script=""): - tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount) - if (sign): - signed = node.signrawtransaction(tx_to_witness) - assert("errors" not in signed or len(["errors"]) == 0) - return node.sendrawtransaction(signed["hex"]) - else: - if (insert_redeem_script): - tx = FromHex(CTransaction(), tx_to_witness) - tx.vin[0].scriptSig += CScript([hex_str_to_bytes(insert_redeem_script)]) - tx_to_witness = ToHex(tx) - - return node.sendrawtransaction(tx_to_witness) - def getutxo(txid): utxo = {} utxo["vout"] = 0 @@ -78,9 +40,9 @@ class SegWitTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. - self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy"], - ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy"], - ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-vbparams=segwit:0:999999999999", "-addresstype=legacy"]] + self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"], + ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"], + ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"]] def setup_network(self): super().setup_network() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 93af0037e9..642ef98a27 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -4,8 +4,24 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" +from .address import ( + key_to_p2sh_p2wpkh, + key_to_p2wpkh, + script_to_p2sh_p2wsh, + script_to_p2wsh, +) from .mininode import * -from .script import CScript, OP_TRUE, OP_CHECKSIG, OP_RETURN +from .script import ( + CScript, + OP_0, + OP_1, + OP_CHECKMULTISIG, + OP_CHECKSIG, + OP_RETURN, + OP_TRUE, + hash160, +) +from .util import assert_equal # Create a block (with regtest difficulty) def create_block(hashprev, coinbase, nTime=None): @@ -108,3 +124,49 @@ def get_legacy_sigopcount_tx(tx, fAccurate=True): # scriptSig might be of type bytes, so convert to CScript for the moment count += CScript(j.scriptSig).GetSigOpCount(fAccurate) return count + +# Create a scriptPubKey corresponding to either a P2WPKH output for the +# given pubkey, or a P2WSH output of a 1-of-1 multisig for the given +# pubkey. Returns the hex encoding of the scriptPubKey. +def witness_script(use_p2wsh, pubkey): + if (use_p2wsh == False): + # P2WPKH instead + pubkeyhash = hash160(hex_str_to_bytes(pubkey)) + pkscript = CScript([OP_0, pubkeyhash]) + else: + # 1-of-1 multisig + witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) + scripthash = sha256(witness_program) + pkscript = CScript([OP_0, scripthash]) + return bytes_to_hex_str(pkscript) + +# Return a transaction (in hex) that spends the given utxo to a segwit output, +# optionally wrapping the segwit output using P2SH. +def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): + if use_p2wsh: + program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) + addr = script_to_p2sh_p2wsh(program) if encode_p2sh else script_to_p2wsh(program) + else: + addr = key_to_p2sh_p2wpkh(pubkey) if encode_p2sh else key_to_p2wpkh(pubkey) + if not encode_p2sh: + assert_equal(node.validateaddress(addr)['scriptPubKey'], witness_script(use_p2wsh, pubkey)) + return node.createrawtransaction([utxo], {addr: amount}) + +# Create a transaction spending a given utxo to a segwit output corresponding +# to the given pubkey: use_p2wsh determines whether to use P2WPKH or P2WSH; +# encode_p2sh determines whether to wrap in P2SH. +# sign=True will have the given node sign the transaction. +# insert_redeem_script will be added to the scriptSig, if given. +def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=True, insert_redeem_script=""): + tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount) + if (sign): + signed = node.signrawtransaction(tx_to_witness) + assert("errors" not in signed or len(["errors"]) == 0) + return node.sendrawtransaction(signed["hex"]) + else: + if (insert_redeem_script): + tx = FromHex(CTransaction(), tx_to_witness) + tx.vin[0].scriptSig += CScript([hex_str_to_bytes(insert_redeem_script)]) + tx_to_witness = ToHex(tx) + + return node.sendrawtransaction(tx_to_witness) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index e42f3e60c2..f8d66def64 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -62,6 +62,7 @@ class BitcoinTestFramework(): self.setup_clean_chain = False self.nodes = [] self.mocktime = 0 + self.supports_cli = False self.set_test_params() assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" @@ -91,6 +92,8 @@ class BitcoinTestFramework(): help="Location of the test framework config file") parser.add_option("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") + parser.add_option("--usecli", dest="usecli", default=False, action="store_true", + help="use bitcoin-cli instead of RPC for all commands") self.add_options(parser) (self.options, self.args) = parser.parse_args() @@ -113,6 +116,8 @@ class BitcoinTestFramework(): success = TestStatus.FAILED try: + if self.options.usecli and not self.supports_cli: + raise SkipTest("--usecli specified but test does not support using CLI") self.setup_chain() self.setup_network() self.run_test() @@ -213,20 +218,20 @@ class BitcoinTestFramework(): assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir)) + self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, use_cli=self.options.usecli)) - def start_node(self, i, extra_args=None, stderr=None): + def start_node(self, i, *args, **kwargs): """Start a bitcoind""" node = self.nodes[i] - node.start(extra_args, stderr) + node.start(*args, **kwargs) node.wait_for_rpc_connection() if self.options.coveragedir is not None: coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc) - def start_nodes(self, extra_args=None): + def start_nodes(self, extra_args=None, *args, **kwargs): """Start multiple bitcoinds""" if extra_args is None: @@ -234,7 +239,7 @@ class BitcoinTestFramework(): assert_equal(len(extra_args), self.num_nodes) try: for i, node in enumerate(self.nodes): - node.start(extra_args[i]) + node.start(extra_args[i], *args, **kwargs) for node in self.nodes: node.wait_for_rpc_connection() except: @@ -266,10 +271,10 @@ class BitcoinTestFramework(): self.stop_node(i) self.start_node(i, extra_args) - def assert_start_raises_init_error(self, i, extra_args=None, expected_msg=None): + def assert_start_raises_init_error(self, i, extra_args=None, expected_msg=None, *args, **kwargs): with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: try: - self.start_node(i, extra_args, stderr=log_stderr) + self.start_node(i, extra_args, stderr=log_stderr, *args, **kwargs) self.stop_node(i) except Exception as e: assert 'bitcoind exited' in str(e) # node must have shutdown diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index a9248c764e..7c784d8840 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -10,6 +10,7 @@ import http.client import json import logging import os +import re import subprocess import time @@ -22,6 +23,9 @@ from .util import ( p2p_port, ) +# For Python 3.4 compatibility +JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) + BITCOIND_PROC_WAIT_TIMEOUT = 60 class TestNode(): @@ -38,7 +42,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir): + def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir, use_cli=False): self.index = i self.datadir = os.path.join(dirname, "node" + str(i)) self.rpchost = rpchost @@ -58,6 +62,7 @@ class TestNode(): self.args = [self.binary, "-datadir=" + self.datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=testnode%d" % i] self.cli = TestNodeCLI(os.getenv("BITCOINCLI", "bitcoin-cli"), self.datadir) + self.use_cli = use_cli self.running = False self.process = None @@ -69,17 +74,20 @@ class TestNode(): self.p2ps = [] def __getattr__(self, name): - """Dispatches any unrecognised messages to the RPC connection.""" - assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" - return getattr(self.rpc, name) + """Dispatches any unrecognised messages to the RPC connection or a CLI instance.""" + if self.use_cli: + return getattr(self.cli, name) + else: + assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" + return getattr(self.rpc, name) - def start(self, extra_args=None, stderr=None): + def start(self, extra_args=None, stderr=None, *args, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args if stderr is None: stderr = self.stderr - self.process = subprocess.Popen(self.args + extra_args, stderr=stderr) + self.process = subprocess.Popen(self.args + extra_args, stderr=stderr, *args, **kwargs) self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") @@ -110,10 +118,13 @@ class TestNode(): raise AssertionError("Unable to connect to bitcoind") def get_wallet_rpc(self, wallet_name): - assert self.rpc_connected - assert self.rpc - wallet_path = "wallet/%s" % wallet_name - return self.rpc / wallet_path + if self.use_cli: + return self.cli("-rpcwallet={}".format(wallet_name)) + else: + assert self.rpc_connected + assert self.rpc + wallet_path = "wallet/%s" % wallet_name + return self.rpc / wallet_path def stop_node(self): """Stop the node.""" @@ -187,6 +198,16 @@ class TestNode(): p.peer_disconnect() del self.p2ps[:] +class TestNodeCLIAttr: + def __init__(self, cli, command): + self.cli = cli + self.command = command + + def __call__(self, *args, **kwargs): + return self.cli.send_cli(self.command, *args, **kwargs) + + def get_request(self, *args, **kwargs): + return lambda: self(*args, **kwargs) class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" @@ -196,17 +217,26 @@ class TestNodeCLI(): self.binary = binary self.datadir = datadir self.input = None + self.log = logging.getLogger('TestFramework.bitcoincli') def __call__(self, *args, input=None): # TestNodeCLI is callable with bitcoin-cli command-line args - self.args = [str(arg) for arg in args] - self.input = input - return self + cli = TestNodeCLI(self.binary, self.datadir) + cli.args = [str(arg) for arg in args] + cli.input = input + return cli def __getattr__(self, command): - def dispatcher(*args, **kwargs): - return self.send_cli(command, *args, **kwargs) - return dispatcher + return TestNodeCLIAttr(self, command) + + def batch(self, requests): + results = [] + for request in requests: + try: + results.append(dict(result=request())) + except JSONRPCException as e: + results.append(dict(error=e)) + return results def send_cli(self, command, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" @@ -218,10 +248,18 @@ class TestNodeCLI(): if named_args: p_args += ["-named"] p_args += [command] + pos_args + named_args + self.log.debug("Running bitcoin-cli command: %s" % command) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() if returncode: + match = re.match(r'error code: ([-0-9]+)\nerror message:\n(.*)', cli_stderr) + if match: + code, message = match.groups() + raise JSONRPCException(dict(code=int(code), message=message)) # Ignore cli_stdout, raise with cli_stderr raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) - return json.loads(cli_stdout, parse_float=decimal.Decimal) + try: + return json.loads(cli_stdout, parse_float=decimal.Decimal) + except JSONDecodeError: + return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 664dc6d0af..7fdc171332 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -390,7 +390,7 @@ def sync_chain(rpc_connections, *, wait=1, timeout=60): timeout -= wait raise AssertionError("Chain sync failed: Best block hashes don't match") -def sync_mempools(rpc_connections, *, wait=1, timeout=60): +def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): """ Wait until everybody has the same transactions in their memory pools @@ -402,6 +402,9 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60): if set(rpc_connections[i].getrawmempool()) == pool: num_match = num_match + 1 if num_match == len(rpc_connections): + if flush_scheduler: + for r in rpc_connections: + r.syncwithvalidationinterfacequeue() return time.sleep(wait) timeout -= wait diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2d5ea84814..b8b6ee98bf 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -94,6 +94,7 @@ BASE_SCRIPTS= [ 'mempool_reorg.py', 'mempool_persist.py', 'multiwallet.py', + 'multiwallet.py --usecli', 'httpbasics.py', 'multi_rpc.py', 'proxy_test.py', @@ -271,6 +272,7 @@ def main(): sys.exit(0) check_script_list(config["environment"]["SRCDIR"]) + check_script_prefixes() if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) @@ -311,9 +313,9 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove # Populate cache try: subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) - except Exception as e: - print(e.output) - raise e + except subprocess.CalledProcessError as e: + sys.stdout.buffer.write(e.output) + raise #Run Tests job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) @@ -469,6 +471,28 @@ class TestResult(): return self.status != "Failed" +def check_script_prefixes(): + """Check that no more than `EXPECTED_VIOLATION_COUNT` of the + test scripts don't start with one of the allowed name prefixes.""" + EXPECTED_VIOLATION_COUNT = 77 + + # LEEWAY is provided as a transition measure, so that pull-requests + # that introduce new tests that don't conform with the naming + # convention don't immediately cause the tests to fail. + LEEWAY = 10 + + good_prefixes_re = re.compile("(example|feature|interface|mempool|mining|p2p|rpc|wallet)_") + bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] + + if len(bad_script_names) < EXPECTED_VIOLATION_COUNT: + print("{}HURRAY!{} Number of functional tests violating naming convention reduced!".format(BOLD[1], BOLD[0])) + print("Consider reducing EXPECTED_VIOLATION_COUNT from %d to %d" % (EXPECTED_VIOLATION_COUNT, len(bad_script_names))) + elif len(bad_script_names) > EXPECTED_VIOLATION_COUNT: + print("INFO: %d tests not meeting naming conventions (expected %d):" % (len(bad_script_names), EXPECTED_VIOLATION_COUNT)) + print(" %s" % ("\n ".join(sorted(bad_script_names)))) + assert len(bad_script_names) <= EXPECTED_VIOLATION_COUNT + LEEWAY, "Too many tests not following naming convention! (%d found, expected: <= %d)" % (len(bad_script_names), EXPECTED_VIOLATION_COUNT) + + def check_script_list(src_dir): """Check scripts directory. diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 86b268094c..77f90ffb81 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -69,7 +69,7 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): class WalletDumpTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-keypool=90", "-addresstype=legacy"]] + self.extra_args = [["-keypool=90", "-addresstype=legacy", "-deprecatedrpc=addwitnessaddress"]] def setup_network(self, split=False): # Use 1 minute timeout because the initial getnewaddress RPC can take diff --git a/test/functional/wallet-encryption.py b/test/functional/wallet-encryption.py index 452e8ec291..3c927ee484 100755 --- a/test/functional/wallet-encryption.py +++ b/test/functional/wallet-encryption.py @@ -10,6 +10,8 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, + assert_greater_than, + assert_greater_than_or_equal, ) class WalletEncryptionTest(BitcoinTestFramework): @@ -56,6 +58,23 @@ class WalletEncryptionTest(BitcoinTestFramework): assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) self.nodes[0].walletpassphrase(passphrase2, 10) assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + self.nodes[0].walletlock() + + # Test timeout bounds + assert_raises_rpc_error(-8, "Timeout cannot be negative.", self.nodes[0].walletpassphrase, passphrase2, -10) + # Check the timeout + # Check a time less than the limit + expected_time = int(time.time()) + (1 << 30) - 600 + self.nodes[0].walletpassphrase(passphrase2, (1 << 30) - 600) + actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] + assert_greater_than_or_equal(actual_time, expected_time) + assert_greater_than(expected_time + 5, actual_time) # 5 second buffer + # Check a time greater than the limit + expected_time = int(time.time()) + (1 << 30) - 1 + self.nodes[0].walletpassphrase(passphrase2, (1 << 33)) + actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] + assert_greater_than_or_equal(actual_time, expected_time) + assert_greater_than(expected_time + 5, actual_time) # 5 second buffer if __name__ == '__main__': WalletEncryptionTest().main() diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 0471f2afb3..08afb87894 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -10,7 +10,7 @@ transactions are still available. - restart node 0 with zapwallettxes and persistmempool, and verify that both the confirmed and the unconfirmed transactions are still available. -- restart node 0 with just zapwallettxed and verify that the confirmed +- restart node 0 with just zapwallettxes and verify that the confirmed transactions are still available, but that the unconfirmed transaction has been zapped. """ |