diff options
Diffstat (limited to 'src')
109 files changed, 2469 insertions, 1529 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 89dd4c8f77..cb88171348 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -98,6 +98,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + fs.h \ httprpc.h \ httpserver.h \ indirectmap.h \ @@ -122,6 +123,7 @@ BITCOIN_CORE_H = \ protocol.h \ random.h \ reverselock.h \ + rpc/blockchain.h \ rpc/client.h \ rpc/protocol.h \ rpc/server.h \ @@ -155,6 +157,7 @@ BITCOIN_CORE_H = \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ + wallet/feebumper.h \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ @@ -229,6 +232,7 @@ libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/crypter.cpp \ wallet/db.cpp \ + wallet/feebumper.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ @@ -327,6 +331,7 @@ libbitcoin_util_a_SOURCES = \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ + fs.cpp \ random.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ @@ -459,6 +464,14 @@ DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) + +config/bitcoin-config.h: config/stamp-h1 + @$(MAKE) -C $(top_builddir) $(subdir)/$(@) +config/stamp-h1: $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in $(top_builddir)/config.status + $(AM_V_at)$(MAKE) -C $(top_builddir) $(subdir)/$(@) +$(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) + $(AM_V_at)$(MAKE) -C $(top_srcdir) $(subdir)/config/bitcoin-config.h.in + clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index edc3c4b292..48411f29ec 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -122,6 +122,7 @@ QT_MOC_CPP = \ qt/moc_bitcoinamountfield.cpp \ qt/moc_bitcoingui.cpp \ qt/moc_bitcoinunits.cpp \ + qt/moc_callback.cpp \ qt/moc_clientmodel.cpp \ qt/moc_coincontroldialog.cpp \ qt/moc_coincontroltreewidget.cpp \ @@ -167,6 +168,7 @@ BITCOIN_MM = \ QT_MOC = \ qt/bitcoin.moc \ qt/bitcoinamountfield.moc \ + qt/callback.moc \ qt/intro.moc \ qt/overviewpage.moc \ qt/rpcconsole.moc @@ -189,6 +191,7 @@ BITCOIN_QT_H = \ qt/bitcoinamountfield.h \ qt/bitcoingui.h \ qt/bitcoinunits.h \ + qt/callback.h \ qt/clientmodel.h \ qt/coincontroldialog.h \ qt/coincontroltreewidget.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 194fc0c249..10cb7e775a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -97,7 +97,7 @@ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) + $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 224d3921cc..a3743cd0d4 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -8,13 +8,13 @@ #include "addrman.h" #include "chainparams.h" #include "clientversion.h" +#include "fs.h" #include "hash.h" #include "random.h" #include "streams.h" #include "tinyformat.h" #include "util.h" -#include <boost/filesystem.hpp> CBanDB::CBanDB() { @@ -36,8 +36,8 @@ bool CBanDB::Write(const banmap_t& banSet) ssBanlist << hash; // open temp output file, and associate with CAutoFile - boost::filesystem::path pathTmp = GetDataDir() / tmpfn; - FILE *file = fopen(pathTmp.string().c_str(), "wb"); + fs::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) return error("%s: Failed to open file %s", __func__, pathTmp.string()); @@ -62,13 +62,13 @@ bool CBanDB::Write(const banmap_t& banSet) bool CBanDB::Read(banmap_t& banSet) { // open input file, and associate with CAutoFile - FILE *file = fopen(pathBanlist.string().c_str(), "rb"); + FILE *file = fsbridge::fopen(pathBanlist, "rb"); CAutoFile filein(file, SER_DISK, CLIENT_VERSION); if (filein.IsNull()) return error("%s: Failed to open file %s", __func__, pathBanlist.string()); // use file size to size memory buffer - uint64_t fileSize = boost::filesystem::file_size(pathBanlist); + uint64_t fileSize = fs::file_size(pathBanlist); uint64_t dataSize = 0; // Don't try to resize to a negative number if file is small if (fileSize >= sizeof(uint256)) @@ -133,8 +133,8 @@ bool CAddrDB::Write(const CAddrMan& addr) ssPeers << hash; // open temp output file, and associate with CAutoFile - boost::filesystem::path pathTmp = GetDataDir() / tmpfn; - FILE *file = fopen(pathTmp.string().c_str(), "wb"); + fs::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) return error("%s: Failed to open file %s", __func__, pathTmp.string()); @@ -159,13 +159,13 @@ bool CAddrDB::Write(const CAddrMan& addr) bool CAddrDB::Read(CAddrMan& addr) { // open input file, and associate with CAutoFile - FILE *file = fopen(pathAddr.string().c_str(), "rb"); + FILE *file = fsbridge::fopen(pathAddr, "rb"); CAutoFile filein(file, SER_DISK, CLIENT_VERSION); if (filein.IsNull()) return error("%s: Failed to open file %s", __func__, pathAddr.string()); // use file size to size memory buffer - uint64_t fileSize = boost::filesystem::file_size(pathAddr); + uint64_t fileSize = fs::file_size(pathAddr); uint64_t dataSize = 0; // Don't try to resize to a negative number if file is small if (fileSize >= sizeof(uint256)) diff --git a/src/addrdb.h b/src/addrdb.h index ab985b10cb..c3d509bd3a 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H +#include "fs.h" #include "serialize.h" #include <string> #include <map> -#include <boost/filesystem/path.hpp> class CSubNet; class CAddrMan; @@ -80,7 +80,7 @@ typedef std::map<CSubNet, CBanEntry> banmap_t; class CAddrDB { private: - boost::filesystem::path pathAddr; + fs::path pathAddr; public: CAddrDB(); bool Write(const CAddrMan& addr); @@ -92,7 +92,7 @@ public: class CBanDB { private: - boost::filesystem::path pathBanlist; + fs::path pathBanlist; public: CBanDB(); bool Write(const banmap_t& banSet); diff --git a/src/addrman.cpp b/src/addrman.cpp index 33a623c1f2..4a408b9beb 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -233,7 +233,7 @@ void CAddrMan::Good_(const CService& addr, int64_t nTime) if (nUBucket == -1) return; - LogPrint("addrman", "Moving %s to tried\n", addr.ToString()); + LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); // move nId to the tried tables MakeTried(info, nId); diff --git a/src/addrman.h b/src/addrman.h index bb45ac91ea..70d907488f 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -447,7 +447,7 @@ public: } } if (nLost + nLostUnk > 0) { - LogPrint("addrman", "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); + LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); } Check(); @@ -512,8 +512,9 @@ public: Check(); fRet |= Add_(addr, source, nTimePenalty); Check(); - if (fRet) - LogPrint("addrman", "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); + if (fRet) { + LogPrint(BCLog::ADDRMAN, "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); + } return fRet; } @@ -526,8 +527,9 @@ public: for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; Check(); - if (nAdd) - LogPrint("addrman", "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + if (nAdd) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + } return nAdd > 0; } diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 230e4ca773..c6c932454a 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -22,7 +22,7 @@ static void DeserializeBlockTest(benchmark::State& state) CDataStream stream((const char*)block_bench::block413567, (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], SER_NETWORK, PROTOCOL_VERSION); - char a; + char a = '\0'; stream.write(&a, 1); // Prevent compaction while (state.KeepRunning()) { @@ -37,7 +37,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) CDataStream stream((const char*)block_bench::block413567, (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], SER_NETWORK, PROTOCOL_VERSION); - char a; + char a = '\0'; stream.write(&a, 1); // Prevent compaction Consensus::Params params = Params(CBaseChainParams::MAIN).GetConsensus(); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 06882f1514..42891f345b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -48,7 +48,7 @@ static void CoinSelection(benchmark::State& state) addCoin(1000 * COIN, wallet, vCoins); addCoin(3 * COIN, wallet, vCoins); - std::set<std::pair<const CWalletTx*, unsigned int> > setCoinsRet; + std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool success = wallet.SelectCoinsMinConf(1003 * COIN, 1, 6, 0, vCoins, setCoinsRet, nValueRet); assert(success); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index ed8ca7e14c..5edd43d41e 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,12 +9,12 @@ #include "chainparamsbase.h" #include "clientversion.h" +#include "fs.h" #include "rpc/client.h" #include "rpc/protocol.h" #include "util.h" #include "utilstrencodings.h" -#include <boost/filesystem/operations.hpp> #include <stdio.h> #include <event2/buffer.h> @@ -96,7 +96,7 @@ static int AppInitRPC(int argc, char* argv[]) } return EXIT_SUCCESS; } - if (!boost::filesystem::is_directory(GetDataDir(false))) { + if (!fs::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", GetArg("-datadir", "").c_str()); return EXIT_FAILURE; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 61e0eb74e6..83b855cbcf 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -242,6 +242,9 @@ static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strIn std::vector<std::string> vStrInputParts; boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + if (vStrInputParts.size() != 2) + throw std::runtime_error("TX output missing or too many separators"); + // Extract and validate VALUE CAmount value = ExtractAndValidateValue(vStrInputParts[0]); @@ -264,6 +267,9 @@ static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& str std::vector<std::string> vStrInputParts; boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + if (vStrInputParts.size() < 2 || vStrInputParts.size() > 3) + throw std::runtime_error("TX output missing or too many separators"); + // Extract and validate VALUE CAmount value = ExtractAndValidateValue(vStrInputParts[0]); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a3d02afcdb..31680a8ec7 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -10,6 +10,7 @@ #include "chainparams.h" #include "clientversion.h" #include "compat.h" +#include "fs.h" #include "rpc/server.h" #include "init.h" #include "noui.h" @@ -20,7 +21,6 @@ #include "utilstrencodings.h" #include <boost/algorithm/string/predicate.hpp> -#include <boost/filesystem.hpp> #include <boost/thread.hpp> #include <stdio.h> @@ -97,7 +97,7 @@ bool AppInit(int argc, char* argv[]) try { - if (!boost::filesystem::is_directory(GetDataDir(false))) + if (!fs::is_directory(GetDataDir(false))) { fprintf(stderr, "Error: Specified data directory \"%s\" does not exist.\n", GetArg("-datadir", "").c_str()); return false; diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 90120b82eb..ee2c654980 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -164,7 +164,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c break; } - LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); + LogPrint(BCLog::CMPCTBLOCK, "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION)); return READ_STATUS_OK; } @@ -209,10 +209,11 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< return READ_STATUS_CHECKBLOCK_FAILED; } - LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); + LogPrint(BCLog::CMPCTBLOCK, "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size()); if (vtx_missing.size() < 5) { - for (const auto& tx : vtx_missing) - LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); + for (const auto& tx : vtx_missing) { + LogPrint(BCLog::CMPCTBLOCK, "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString()); + } } return READ_STATUS_OK; diff --git a/src/checkqueue.h b/src/checkqueue.h index ea12df66dd..63c104c02a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H +#include "sync.h" + #include <algorithm> #include <vector> diff --git a/src/clientversion.h b/src/clientversion.h index 69154d546d..3d5392619b 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -7,29 +7,13 @@ #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" -#else - -/** - * client versioning and copyright year - */ - -//! These need to be macros, as clientversion.cpp's and bitcoin*-res.rc's voodoo requires it -#define CLIENT_VERSION_MAJOR 0 -#define CLIENT_VERSION_MINOR 14 -#define CLIENT_VERSION_REVISION 99 -#define CLIENT_VERSION_BUILD 0 - -//! Set to true for release, false for prerelease or test build -#define CLIENT_VERSION_IS_RELEASE false - -/** - * Copyright year (2009-this) - * Todo: update this when changing our copyright comments in the source - */ -#define COPYRIGHT_YEAR 2017 - #endif //HAVE_CONFIG_H +// Check that required client information is defined +#if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_REVISION) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) +#error Client version information missing: version is not defined by bitcoin-config.h or in any other way +#endif + /** * Converts the parameter X to a string after macro replacement on X has been performed. * Don't merge these into one macro! diff --git a/src/coins.h b/src/coins.h index 8ee49b33ae..0cb8519df8 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" diff --git a/src/cuckoocache.h b/src/cuckoocache.h index ff47e9776b..5837549455 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -154,7 +154,7 @@ public: * @tparam Element should be a movable and copyable type * @tparam Hash should be a function/callable which takes a template parameter * hash_select and an Element and extracts a hash from it. Should return - * high-entropy hashes for `Hash h; h<0>(e) ... h<7>(e)`. + * high-entropy uint32_t hashes for `Hash h; h<0>(e) ... h<7>(e)`. */ template <typename Element, typename Hash> class cache @@ -193,12 +193,6 @@ private: */ uint32_t epoch_size; - /** hash_mask should be set to appropriately mask out a hash such that every - * masked hash is [0,size), eg, if floor(log2(size)) == 20, then hash_mask - * should be (1<<20)-1 - */ - uint32_t hash_mask; - /** depth_limit determines how many elements insert should try to replace. * Should be set to log2(n)*/ uint8_t depth_limit; @@ -217,14 +211,14 @@ private: */ inline std::array<uint32_t, 8> compute_hashes(const Element& e) const { - return {{hash_function.template operator()<0>(e) & hash_mask, - hash_function.template operator()<1>(e) & hash_mask, - hash_function.template operator()<2>(e) & hash_mask, - hash_function.template operator()<3>(e) & hash_mask, - hash_function.template operator()<4>(e) & hash_mask, - hash_function.template operator()<5>(e) & hash_mask, - hash_function.template operator()<6>(e) & hash_mask, - hash_function.template operator()<7>(e) & hash_mask}}; + return {{(uint32_t)((hash_function.template operator()<0>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<1>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<2>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<3>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<4>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<5>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<6>(e) * (uint64_t)size) >> 32), + (uint32_t)((hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}}; } /* end @@ -305,7 +299,7 @@ public: } /** setup initializes the container to store no more than new_size - * elements. setup rounds down to a power of two size. + * elements. * * setup should only be called once. * @@ -316,8 +310,7 @@ public: { // depth_limit must be at least one otherwise errors can occur. depth_limit = static_cast<uint8_t>(std::log2(static_cast<float>(std::max((uint32_t)2, new_size)))); - size = 1 << depth_limit; - hash_mask = size-1; + size = std::max<uint32_t>(2, new_size); table.resize(size); collection_flags.setup(size); epoch_flags.resize(size); diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index cd441add4b..3d2098c059 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,11 +4,10 @@ #include "dbwrapper.h" +#include "fs.h" #include "util.h" #include "random.h" -#include <boost/filesystem.hpp> - #include <leveldb/cache.h> #include <leveldb/env.h> #include <leveldb/filter_policy.h> @@ -21,8 +20,9 @@ public: // This code is adapted from posix_logger.h, which is why it is using vsprintf. // Please do not do this in normal code virtual void Logv(const char * format, va_list ap) override { - if (!LogAcceptCategory("leveldb")) + if (!LogAcceptCategory(BCLog::LEVELDB)) { return; + } char buffer[500]; for (int iter = 0; iter < 2; iter++) { char* base; @@ -90,7 +90,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) { penv = NULL; readoptions.verify_checksums = true; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 414df76a7c..b13e98b7a4 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -6,14 +6,13 @@ #define BITCOIN_DBWRAPPER_H #include "clientversion.h" +#include "fs.h" #include "serialize.h" #include "streams.h" #include "util.h" #include "utilstrencodings.h" #include "version.h" -#include <boost/filesystem/path.hpp> - #include <leveldb/db.h> #include <leveldb/write_batch.h> @@ -195,7 +194,7 @@ public: * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR * with a zero'd byte array. */ - CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); ~CDBWrapper(); template <typename K, typename V> diff --git a/src/fs.cpp b/src/fs.cpp new file mode 100644 index 0000000000..6f2b768de3 --- /dev/null +++ b/src/fs.cpp @@ -0,0 +1,17 @@ +#include "fs.h" + +#include <boost/filesystem.hpp> + +namespace fsbridge { + +FILE *fopen(const fs::path& p, const char *mode) +{ + return ::fopen(p.string().c_str(), mode); +} + +FILE *freopen(const fs::path& p, const char *mode, FILE *stream) +{ + return ::freopen(p.string().c_str(), mode, stream); +} + +} // fsbridge diff --git a/src/fs.h b/src/fs.h new file mode 100644 index 0000000000..585cbf9c38 --- /dev/null +++ b/src/fs.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FS_H +#define BITCOIN_FS_H + +#include <stdio.h> +#include <string> + +#include <boost/filesystem.hpp> +#include <boost/filesystem/fstream.hpp> +#include <boost/filesystem/detail/utf8_codecvt_facet.hpp> + +/** Filesystem operations and types */ +namespace fs = boost::filesystem; + +/** Bridge operations to C stdio */ +namespace fsbridge { + FILE *fopen(const fs::path& p, const char *mode); + FILE *freopen(const fs::path& p, const char *mode, FILE *stream); +}; + +#endif diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 8ac6925acd..21c64c5c83 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -233,7 +233,7 @@ static bool InitRPCAuthentication() bool StartHTTPRPC() { - LogPrint("rpc", "Starting HTTP RPC server\n"); + LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) return false; @@ -247,12 +247,12 @@ bool StartHTTPRPC() void InterruptHTTPRPC() { - LogPrint("rpc", "Interrupting HTTP RPC server\n"); + LogPrint(BCLog::RPC, "Interrupting HTTP RPC server\n"); } void StopHTTPRPC() { - LogPrint("rpc", "Stopping HTTP RPC server\n"); + LogPrint(BCLog::RPC, "Stopping HTTP RPC server\n"); UnregisterHTTPHandler("/", true); if (httpRPCTimerInterface) { RPCUnsetTimerInterface(httpRPCTimerInterface); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 799de1c2cb..e7df23295d 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -213,7 +213,7 @@ static bool InitHTTPAllowList() std::string strAllowed; for (const CSubNet& subnet : rpc_allow_subnets) strAllowed += subnet.ToString() + " "; - LogPrint("http", "Allowing HTTP connections from: %s\n", strAllowed); + LogPrint(BCLog::HTTP, "Allowing HTTP connections from: %s\n", strAllowed); return true; } @@ -243,7 +243,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) { std::unique_ptr<HTTPRequest> hreq(new HTTPRequest(req)); - LogPrint("http", "Received a %s request for %s from %s\n", + LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); // Early address-based allow check @@ -293,7 +293,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) /** Callback to reject HTTP requests after shutdown. */ static void http_reject_request_cb(struct evhttp_request* req, void*) { - LogPrint("http", "Rejecting request while shutting down\n"); + LogPrint(BCLog::HTTP, "Rejecting request while shutting down\n"); evhttp_send_error(req, HTTP_SERVUNAVAIL, NULL); } @@ -301,10 +301,10 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) static bool ThreadHTTP(struct event_base* base, struct evhttp* http) { RenameThread("bitcoin-http"); - LogPrint("http", "Entering http event loop\n"); + LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() - LogPrint("http", "Exited http event loop\n"); + LogPrint(BCLog::HTTP, "Exited http event loop\n"); return event_base_got_break(base) == 0; } @@ -336,7 +336,7 @@ static bool HTTPBindAddresses(struct evhttp* http) // Bind addresses for (std::vector<std::pair<std::string, uint16_t> >::iterator i = endpoints.begin(); i != endpoints.end(); ++i) { - LogPrint("http", "Binding RPC on address %s port %i\n", i->first, i->second); + LogPrint(BCLog::HTTP, "Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? NULL : i->first.c_str(), i->second); if (bind_handle) { boundSockets.push_back(bind_handle); @@ -364,7 +364,7 @@ static void libevent_log_cb(int severity, const char *msg) if (severity >= EVENT_LOG_WARN) // Log warn messages and higher without debug category LogPrintf("libevent: %s\n", msg); else - LogPrint("libevent", "libevent: %s\n", msg); + LogPrint(BCLog::LIBEVENT, "libevent: %s\n", msg); } bool InitHTTPServer() @@ -384,14 +384,13 @@ bool InitHTTPServer() // Redirect libevent's logging to our own log event_set_log_callback(&libevent_log_cb); -#if LIBEVENT_VERSION_NUMBER >= 0x02010100 - // If -debug=libevent, set full libevent debugging. - // Otherwise, disable all libevent debugging. - if (LogAcceptCategory("libevent")) - event_enable_debug_logging(EVENT_DBG_ALL); - else - event_enable_debug_logging(EVENT_DBG_NONE); -#endif + // Update libevent's log handling. Returns false if our version of + // libevent doesn't support debug logging, in which case we should + // clear the BCLog::LIBEVENT flag. + if (!UpdateHTTPServerLogging(logCategories & BCLog::LIBEVENT)) { + logCategories &= ~BCLog::LIBEVENT; + } + #ifdef WIN32 evthread_use_windows_threads(); #else @@ -424,7 +423,7 @@ bool InitHTTPServer() return false; } - LogPrint("http", "Initialized HTTP server\n"); + LogPrint(BCLog::HTTP, "Initialized HTTP server\n"); int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); @@ -434,12 +433,26 @@ bool InitHTTPServer() return true; } +bool UpdateHTTPServerLogging(bool enable) { +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + if (enable) { + event_enable_debug_logging(EVENT_DBG_ALL); + } else { + event_enable_debug_logging(EVENT_DBG_NONE); + } + return true; +#else + // Can't update libevent logging if version < 02010100 + return false; +#endif +} + std::thread threadHTTP; std::future<bool> threadResult; bool StartHTTPServer() { - LogPrint("http", "Starting HTTP server\n"); + LogPrint(BCLog::HTTP, "Starting HTTP server\n"); int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); std::packaged_task<bool(event_base*, evhttp*)> task(ThreadHTTP); @@ -455,7 +468,7 @@ bool StartHTTPServer() void InterruptHTTPServer() { - LogPrint("http", "Interrupting HTTP server\n"); + LogPrint(BCLog::HTTP, "Interrupting HTTP server\n"); if (eventHTTP) { // Unlisten sockets for (evhttp_bound_socket *socket : boundSockets) { @@ -470,15 +483,15 @@ void InterruptHTTPServer() void StopHTTPServer() { - LogPrint("http", "Stopping HTTP server\n"); + LogPrint(BCLog::HTTP, "Stopping HTTP server\n"); if (workQueue) { - LogPrint("http", "Waiting for HTTP worker threads to exit\n"); + LogPrint(BCLog::HTTP, "Waiting for HTTP worker threads to exit\n"); workQueue->WaitExit(); delete workQueue; workQueue = nullptr; } if (eventBase) { - LogPrint("http", "Waiting for HTTP event thread to exit\n"); + LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); // Give event loop a few seconds to exit (to send back last RPC responses), then break it // Before this was solved with event_base_loopexit, but that didn't work as expected in // at least libevent 2.0.21 and always introduced a delay. In libevent @@ -499,7 +512,7 @@ void StopHTTPServer() event_base_free(eventBase); eventBase = 0; } - LogPrint("http", "Stopped HTTP server\n"); + LogPrint(BCLog::HTTP, "Stopped HTTP server\n"); } struct event_base* EventBase() @@ -646,7 +659,7 @@ HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { - LogPrint("http", "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); + LogPrint(BCLog::HTTP, "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); } @@ -659,7 +672,7 @@ void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) break; if (i != iend) { - LogPrint("http", "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); + LogPrint(BCLog::HTTP, "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.erase(i); } } diff --git a/src/httpserver.h b/src/httpserver.h index b55b253bea..6be9950682 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -32,6 +32,10 @@ void InterruptHTTPServer(); /** Stop HTTP server */ void StopHTTPServer(); +/** Change logging level for libevent. Removes BCLog::LIBEVENT from logCategories if + * libevent doesn't support debug logging.*/ +bool UpdateHTTPServerLogging(bool enable); + /** Handler for requests to a certain HTTP path */ typedef std::function<bool(HTTPRequest* req, const std::string &)> HTTPRequestHandler; /** Register handler for prefix. diff --git a/src/init.cpp b/src/init.cpp index a76c04d003..f06c9e1100 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -16,6 +16,7 @@ #include "checkpoints.h" #include "compat/sanity.h" #include "consensus/validation.h" +#include "fs.h" #include "httpserver.h" #include "httprpc.h" #include "key.h" @@ -24,9 +25,11 @@ #include "netbase.h" #include "net.h" #include "net_processing.h" +#include "policy/fees.h" #include "policy/policy.h" #include "rpc/server.h" #include "rpc/register.h" +#include "rpc/blockchain.h" #include "script/standard.h" #include "script/sigcache.h" #include "scheduler.h" @@ -55,7 +58,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/bind.hpp> -#include <boost/filesystem.hpp> #include <boost/function.hpp> #include <boost/interprocess/sync/file_lock.hpp> #include <boost/thread.hpp> @@ -211,10 +213,10 @@ void Shutdown() if (fFeeEstimatesInitialized) { - boost::filesystem::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_fileout(fopen(est_path.string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); + fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) - mempool.WriteFeeEstimates(est_fileout); + ::feeEstimator.Write(est_fileout); else LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, est_path.string()); fFeeEstimatesInitialized = false; @@ -249,8 +251,8 @@ void Shutdown() #ifndef WIN32 try { - boost::filesystem::remove(GetPidFile()); - } catch (const boost::filesystem::filesystem_error& e) { + fs::remove(GetPidFile()); + } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove pidfile: %s\n", __func__, e.what()); } #endif @@ -311,7 +313,7 @@ void OnRPCStopped() uiInterface.NotifyBlockTip.disconnect(&RPCNotifyBlockChange); RPCNotifyBlockChange(false, nullptr); cvBlockChange.notify_all(); - LogPrint("rpc", "RPC stopped.\n"); + LogPrint(BCLog::RPC, "RPC stopped.\n"); } void OnRPCPreCommand(const CRPCCommand& cmd) @@ -440,13 +442,9 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified BIP9 deployment (regtest-only)"); } - std::string debugCategories = "addrman, alert, bench, cmpctblock, coindb, db, http, leveldb, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below - if (mode == HMM_BITCOIN_QT) - debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " + - _("If <category> is not supplied or if <category> = 1, output all debugging information.") + _("<category> can be:") + " " + debugCategories + "."); - if (showDebug) - strUsage += HelpMessageOpt("-nodebug", "Turn off debugging messages, same as -debug=0"); + _("If <category> is not supplied or if <category> = 1, output all debugging information.") + " " + _("<category> can be:") + " " + ListLogCategories() + "."); + strUsage += HelpMessageOpt("-debugexclude=<category>", strprintf(_("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."))); strUsage += HelpMessageOpt("-help-debug", _("Show all debugging options (usage: --help -help-debug)")); strUsage += HelpMessageOpt("-logips", strprintf(_("Include IP addresses in debug output (default: %u)"), DEFAULT_LOGIPS)); strUsage += HelpMessageOpt("-logtimestamps", strprintf(_("Prepend debug output with timestamp (default: %u)"), DEFAULT_LOGTIMESTAMPS)); @@ -580,14 +578,14 @@ struct CImportingNow // works correctly. void CleanupBlockRevFiles() { - std::map<std::string, boost::filesystem::path> mapBlockFiles; + std::map<std::string, fs::path> mapBlockFiles; // Glob all blk?????.dat and rev?????.dat files from the blocks directory. // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - boost::filesystem::path blocksdir = GetDataDir() / "blocks"; - for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) { + fs::path blocksdir = GetDataDir() / "blocks"; + for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { if (is_regular_file(*it) && it->path().filename().string().length() == 12 && it->path().filename().string().substr(8,4) == ".dat") @@ -604,7 +602,7 @@ void CleanupBlockRevFiles() // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) // start removing block files. int nContigCounter = 0; - BOOST_FOREACH(const PAIRTYPE(std::string, boost::filesystem::path)& item, mapBlockFiles) { + BOOST_FOREACH(const PAIRTYPE(std::string, fs::path)& item, mapBlockFiles) { if (atoi(item.first) == nContigCounter) { nContigCounter++; continue; @@ -613,7 +611,7 @@ void CleanupBlockRevFiles() } } -void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) +void ThreadImport(std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); RenameThread("bitcoin-loadblk"); @@ -626,7 +624,7 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) int nFile = 0; while (true) { CDiskBlockPos pos(nFile, 0); - if (!boost::filesystem::exists(GetBlockPosFilename(pos, "blk"))) + if (!fs::exists(GetBlockPosFilename(pos, "blk"))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) @@ -643,11 +641,11 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) } // hardcoded $DATADIR/bootstrap.dat - boost::filesystem::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (boost::filesystem::exists(pathBootstrap)) { - FILE *file = fopen(pathBootstrap.string().c_str(), "rb"); + fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; + if (fs::exists(pathBootstrap)) { + FILE *file = fsbridge::fopen(pathBootstrap, "rb"); if (file) { - boost::filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; + fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; LogPrintf("Importing bootstrap.dat...\n"); LoadExternalBlockFile(chainparams, file); RenameOver(pathBootstrap, pathBootstrapOld); @@ -657,8 +655,8 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) } // -loadblock= - BOOST_FOREACH(const boost::filesystem::path& path, vImportFiles) { - FILE *file = fopen(path.string().c_str(), "rb"); + BOOST_FOREACH(const fs::path& path, vImportFiles) { + FILE *file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); LoadExternalBlockFile(chainparams, file); @@ -907,13 +905,33 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - - fDebug = mapMultiArgs.count("-debug"); - // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - if (fDebug) { + if (mapMultiArgs.count("-debug") > 0) { + // Special-case: if -debug=0/-nodebug is set, turn off debugging messages const std::vector<std::string>& categories = mapMultiArgs.at("-debug"); - if (GetBoolArg("-nodebug", false) || find(categories.begin(), categories.end(), std::string("0")) != categories.end()) - fDebug = false; + + if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { + for (const auto& cat : categories) { + uint32_t flag = 0; + if (!GetLogCategory(&flag, &cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); + continue; + } + logCategories |= flag; + } + } + } + + // Now remove the logging categories which were explicitly excluded + if (mapMultiArgs.count("-debugexclude") > 0) { + const std::vector<std::string>& excludedCategories = mapMultiArgs.at("-debugexclude"); + for (const auto& cat : excludedCategories) { + uint32_t flag = 0; + if (!GetLogCategory(&flag, &cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); + continue; + } + logCategories &= ~flag; + } } // Check for -debugnet @@ -1118,8 +1136,8 @@ static bool LockDataDirectory(bool probeOnly) std::string strDataDir = GetDataDir().string(); // Make sure only a single Bitcoin process is using the data directory. - boost::filesystem::path pathLockFile = GetDataDir() / ".lock"; - FILE* file = fopen(pathLockFile.string().c_str(), "a"); // empty lock file; created if it doesn't exist. + fs::path pathLockFile = GetDataDir() / ".lock"; + FILE* file = fsbridge::fopen(pathLockFile, "a"); // empty lock file; created if it doesn't exist. if (file) fclose(file); try { @@ -1167,7 +1185,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) #ifndef WIN32 CreatePidFile(GetPidFile(), getpid()); #endif - if (GetBoolArg("-shrinkdebugfile", !fDebug)) { + if (GetBoolArg("-shrinkdebugfile", logCategories == BCLog::NONE)) { // Do this first since it both loads a bunch of debug.log into memory, // and because this needs to happen before any other debug.log printing ShrinkDebugFile(); @@ -1383,7 +1401,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) fReindex = GetBoolArg("-reindex", false); bool fReindexChainState = GetBoolArg("-reindex-chainstate", false); - boost::filesystem::create_directories(GetDataDir() / "blocks"); + fs::create_directories(GetDataDir() / "blocks"); // cache size calculations int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); @@ -1491,7 +1509,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) break; } } catch (const std::exception& e) { - if (fDebug) LogPrintf("%s\n", e.what()); + LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database"); break; } @@ -1529,11 +1547,11 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } LogPrintf(" block index %15dms\n", GetTimeMillis() - nStart); - boost::filesystem::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_filein(fopen(est_path.string().c_str(), "rb"), SER_DISK, CLIENT_VERSION); + fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); // Allowed to fail as this file IS missing on first startup. if (!est_filein.IsNull()) - mempool.ReadFeeEstimates(est_filein); + ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; // ********************************************************* Step 8: load wallet @@ -1585,7 +1603,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (IsArgSet("-blocknotify")) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); - std::vector<boost::filesystem::path> vImportFiles; + std::vector<fs::path> vImportFiles; if (mapMultiArgs.count("-loadblock")) { BOOST_FOREACH(const std::string& strFile, mapMultiArgs.at("-loadblock")) diff --git a/src/merkleblock.h b/src/merkleblock.h index 73cbf670ee..de4c5c8d29 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -23,7 +23,7 @@ * storing a bit for each traversed node, signifying whether the node is the * parent of at least one matched leaf txid (or a matched txid itself). In * case we are at the leaf level, or this bit is 0, its merkle node hash is - * stored, and its children are not explorer further. Otherwise, no hash is + * stored, and its children are not explored further. Otherwise, no hash is * stored, but we recurse into both (or the only) child branch. During * decoding, the same depth-first traversal is performed, consuming bits and * hashes as they written during encoding. diff --git a/src/miner.cpp b/src/miner.cpp index 4fd99c5282..386d75c4be 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -126,6 +126,8 @@ void BlockAssembler::resetBlock() std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn, bool fMineWitnessTx) { + int64_t nTimeStart = GetTimeMicros(); + resetBlock(); pblocktemplate.reset(new CBlockTemplate()); @@ -164,7 +166,11 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc // transaction (which in most cases can be a no-op). fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus()) && fMineWitnessTx; - addPackageTxs(); + int nPackagesSelected = 0; + int nDescendantsUpdated = 0; + addPackageTxs(nPackagesSelected, nDescendantsUpdated); + + int64_t nTime1 = GetTimeMicros(); nLastBlockTx = nBlockTx; nLastBlockSize = nBlockSize; @@ -196,6 +202,9 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); } + int64_t nTime2 = GetTimeMicros(); + + LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart)); return std::move(pblocktemplate); } @@ -269,9 +278,10 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) } } -void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, +int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx) { + int nDescendantsUpdated = 0; BOOST_FOREACH(const CTxMemPool::txiter it, alreadyAdded) { CTxMemPool::setEntries descendants; mempool.CalculateDescendants(it, descendants); @@ -279,6 +289,7 @@ void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alread BOOST_FOREACH(CTxMemPool::txiter desc, descendants) { if (alreadyAdded.count(desc)) continue; + ++nDescendantsUpdated; modtxiter mit = mapModifiedTx.find(desc); if (mit == mapModifiedTx.end()) { CTxMemPoolModifiedEntry modEntry(desc); @@ -291,6 +302,7 @@ void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alread } } } + return nDescendantsUpdated; } // Skip entries in mapTx that are already in a block or are present @@ -331,7 +343,7 @@ void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemP // Each time through the loop, we compare the best transaction in // mapModifiedTxs with the next transaction in the mempool to decide what // transaction package to work on next. -void BlockAssembler::addPackageTxs() +void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated) { // mapModifiedTx will store sorted packages after they are modified // because some of their txs are already in the block @@ -345,6 +357,13 @@ void BlockAssembler::addPackageTxs() CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; + + // Limit the number of attempts to add transactions to the block when it is + // close to full; this is just a simple heuristic to finish quickly if the + // mempool has a lot of entries. + const int64_t MAX_CONSECUTIVE_FAILURES = 1000; + int64_t nConsecutiveFailed = 0; + while (mi != mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty()) { // First try to find a new transaction in mapTx to evaluate. @@ -406,6 +425,14 @@ void BlockAssembler::addPackageTxs() mapModifiedTx.get<ancestor_score>().erase(modit); failedTx.insert(iter); } + + ++nConsecutiveFailed; + + if (nConsecutiveFailed > MAX_CONSECUTIVE_FAILURES && nBlockWeight > + nBlockMaxWeight - 4000) { + // Give up if we're close to full and haven't succeeded in a while + break; + } continue; } @@ -426,6 +453,9 @@ void BlockAssembler::addPackageTxs() continue; } + // This transaction will make it in; reset the failed counter. + nConsecutiveFailed = 0; + // Package can be added. Sort the entries in a valid order. std::vector<CTxMemPool::txiter> sortedEntries; SortForBlock(ancestors, iter, sortedEntries); @@ -436,8 +466,10 @@ void BlockAssembler::addPackageTxs() mapModifiedTx.erase(sortedEntries[i]); } + ++nPackagesSelected; + // Update transactions that depend on each of these - UpdatePackagesForAdded(ancestors, mapModifiedTx); + nDescendantsUpdated += UpdatePackagesForAdded(ancestors, mapModifiedTx); } } diff --git a/src/miner.h b/src/miner.h index a159b05534..1f3c9d652f 100644 --- a/src/miner.h +++ b/src/miner.h @@ -180,8 +180,10 @@ private: void AddToBlock(CTxMemPool::txiter iter); // Methods for how to add transactions to a block. - /** Add transactions based on feerate including unconfirmed ancestors */ - void addPackageTxs(); + /** Add transactions based on feerate including unconfirmed ancestors + * Increments nPackagesSelected / nDescendantsUpdated with corresponding + * statistics from the package selection (for logging statistics). */ + void addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated); // helper functions for addPackageTxs() /** Remove confirmed (inBlock) entries from given set */ @@ -199,8 +201,9 @@ private: /** Sort the package in an order that is valid to appear in a block */ void SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries); /** Add descendants of given transactions to mapModifiedTx with ancestor - * state updated assuming given transactions are inBlock. */ - void UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx); + * state updated assuming given transactions are inBlock. Returns number + * of updated descendants. */ + int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx); }; /** Modify the extranonce in a block */ diff --git a/src/net.cpp b/src/net.cpp index 4c5b04b785..27389d6e0c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -190,7 +190,7 @@ void AdvertiseLocal(CNode *pnode) } if (addrLocal.IsRoutable()) { - LogPrint("net", "AdvertiseLocal: advertising address %s\n", addrLocal.ToString()); + LogPrint(BCLog::NET, "AdvertiseLocal: advertising address %s\n", addrLocal.ToString()); FastRandomContext insecure_rand; pnode->PushAddress(addrLocal, insecure_rand); } @@ -356,7 +356,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo } /// debug print - LogPrint("net", "trying connection %s lastseen=%.1fhrs\n", + LogPrint(BCLog::NET, "trying connection %s lastseen=%.1fhrs\n", pszDest ? pszDest : addrConnect.ToString(), pszDest ? 0.0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0); @@ -423,7 +423,7 @@ void CConnman::DumpBanlist() if (!bandb.Write(banmap)) SetBannedSetDirty(true); - LogPrint("net", "Flushed %d banned node ips/subnets to banlist.dat %dms\n", + LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); } @@ -433,7 +433,7 @@ void CNode::CloseSocketDisconnect() LOCK(cs_hSocket); if (hSocket != INVALID_SOCKET) { - LogPrint("net", "disconnecting peer=%d\n", id); + LogPrint(BCLog::NET, "disconnecting peer=%d\n", id); CloseSocket(hSocket); } } @@ -565,7 +565,7 @@ void CConnman::SweepBanned() { setBanned.erase(it++); setBannedIsDirty = true; - LogPrint("net", "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + LogPrint(BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); } else ++it; @@ -711,7 +711,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete return false; if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogPrint("net", "Oversized message from peer=%i, disconnecting\n", GetId()); + LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); return false; } @@ -1087,7 +1087,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { { if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection - LogPrint("net", "failed to find an eviction candidate - connection dropped (full)\n"); + LogPrint(BCLog::NET, "failed to find an eviction candidate - connection dropped (full)\n"); CloseSocket(hSocket); return; } @@ -1101,7 +1101,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { pnode->fWhitelisted = whitelisted; GetNodeSignals().InitializeNode(pnode, *this); - LogPrint("net", "connection from %s accepted\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); { LOCK(cs_vNodes); @@ -1336,8 +1336,9 @@ void CConnman::ThreadSocketHandler() else if (nBytes == 0) { // socket closed gracefully - if (!pnode->fDisconnect) - LogPrint("net", "socket closed\n"); + if (!pnode->fDisconnect) { + LogPrint(BCLog::NET, "socket closed\n"); + } pnode->CloseSocketDisconnect(); } else if (nBytes < 0) @@ -1375,7 +1376,7 @@ void CConnman::ThreadSocketHandler() { if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) { - LogPrint("net", "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->id); + LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->id); pnode->fDisconnect = true; } else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) @@ -1584,6 +1585,9 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); BOOST_FOREACH(const CDNSSeedData &seed, vSeeds) { + if (interruptNet) { + return; + } if (HaveNameProxy()) { AddOneShot(seed.host); } else { @@ -1601,6 +1605,9 @@ void CConnman::ThreadDNSAddressSeed() found++; } } + if (interruptNet) { + return; + } // TODO: The seed name resolve may fail, yielding an IP of [::], which results in // addrman assigning the same source to results from different seeds. // This should switch to a hard-coded stable dummy IP for each seed name, so that the @@ -1634,7 +1641,7 @@ void CConnman::DumpAddresses() CAddrDB adb; adb.Write(addrman); - LogPrint("net", "Flushed %d addresses to peers.dat %dms\n", + LogPrint(BCLog::NET, "Flushed %d addresses to peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); } @@ -1807,7 +1814,7 @@ void CConnman::ThreadOpenConnections() int randsleep = GetRandInt(FEELER_SLEEP_WINDOW * 1000); if (!interruptNet.sleep_for(std::chrono::milliseconds(randsleep))) return; - LogPrint("net", "Making feeler connection to %s\n", addrConnect.ToString()); + LogPrint(BCLog::NET, "Making feeler connection to %s\n", addrConnect.ToString()); } OpenNetworkConnection(addrConnect, (int)setConnected.size() >= std::min(nMaxConnections - 1, 2), &grant, NULL, false, fFeeler); @@ -2150,9 +2157,7 @@ void Discover(boost::thread_group& threadGroup) void CConnman::SetNetworkActive(bool active) { - if (fDebug) { - LogPrint("net", "SetNetworkActive: %s\n", active); - } + LogPrint(BCLog::NET, "SetNetworkActive: %s\n", active); if (!active) { fNetworkActive = false; @@ -2241,7 +2246,7 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c SetBannedSetDirty(false); // no need to write down, just read data SweepBanned(); // sweep out unused entries - LogPrint("net", "Loaded %d banned node ips/subnets from banlist.dat %dms\n", + LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); @@ -2683,10 +2688,11 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn mapRecvBytesPerMsgCmd[msg] = 0; mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; - if (fLogIPs) - LogPrint("net", "Added connection to %s peer=%d\n", addrName, id); - else - LogPrint("net", "Added connection peer=%d\n", id); + if (fLogIPs) { + LogPrint(BCLog::NET, "Added connection to %s peer=%d\n", addrName, id); + } else { + LogPrint(BCLog::NET, "Added connection peer=%d\n", id); + } } CNode::~CNode() @@ -2713,7 +2719,7 @@ void CNode::AskFor(const CInv& inv) nRequestTime = it->second; else nRequestTime = 0; - LogPrint("net", "askfor %s %d (%s) peer=%d\n", inv.ToString(), nRequestTime, DateTimeStrFormat("%H:%M:%S", nRequestTime/1000000), id); + LogPrint(BCLog::NET, "askfor %s %d (%s) peer=%d\n", inv.ToString(), nRequestTime, DateTimeStrFormat("%H:%M:%S", nRequestTime/1000000), id); // Make sure not to reuse time indexes to keep things in the same order int64_t nNow = GetTimeMicros() - 1000000; @@ -2740,7 +2746,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint("net", "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -32,11 +32,9 @@ #include <arpa/inet.h> #endif -#include <boost/filesystem/path.hpp> #include <boost/foreach.hpp> #include <boost/signals2/signal.hpp> -class CAddrMan; class CScheduler; class CNode; @@ -92,7 +90,7 @@ static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban -typedef int NodeId; +typedef int64_t NodeId; struct AddedNodeInfo { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 71a0a1de22..2e7b99baa4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -256,10 +256,11 @@ void PushNodeVersion(CNode *pnode, CConnman& connman, int64_t nTime) connman.PushMessage(pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, nonce, strSubVersion, nNodeStartingHeight, ::fRelayTxes)); - if (fLogIPs) - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); - else - LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + if (fLogIPs) { + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + } else { + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + } } void InitializeNode(CNode *pnode, CConnman& connman) { @@ -619,7 +620,7 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE unsigned int sz = GetTransactionWeight(*tx); if (sz >= MAX_STANDARD_TX_WEIGHT) { - LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); + LogPrint(BCLog::MEMPOOL, "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; } @@ -631,7 +632,7 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE AddToCompactExtraTransactions(tx); - LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), + LogPrint(BCLog::MEMPOOL, "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } @@ -666,7 +667,7 @@ void EraseOrphansFor(NodeId peer) nErased += EraseOrphanTx(maybeErase->second.tx->GetHash()); } } - if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer=%d\n", nErased, peer); + if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer=%d\n", nErased, peer); } @@ -691,7 +692,7 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRE } // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; - if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); + if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx due to expiration\n", nErased); } while (mapOrphanTransactions.size() > nMaxOrphans) { @@ -743,21 +744,23 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanI recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); } -void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) { - if (nPosInBlock == CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK) - return; - +void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { LOCK(cs_main); std::vector<uint256> vOrphanErase; - // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = *(*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); + + for (const CTransactionRef& ptx : pblock->vtx) { + const CTransaction& tx = *ptx; + + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = *(*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } } } @@ -767,14 +770,16 @@ void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIn BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { nErased += EraseOrphanTx(orphanHash); } - LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } } +// All of the following cache a recent block, and are protected by cs_most_recent_block static CCriticalSection cs_most_recent_block; static std::shared_ptr<const CBlock> most_recent_block; static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block; static uint256 most_recent_block_hash; +static bool fWitnessesPresentInMostRecentCompactBlock; void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); @@ -795,6 +800,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: most_recent_block_hash = hashBlock; most_recent_block = pblock; most_recent_compact_block = pcmpctblock; + fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { @@ -808,7 +814,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: if (state.fPreferHeaderAndIDs && (!fWitnessEnabled || state.fWantsCmpctWitness) && !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { - LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", + LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", hashBlock.ToString(), pnode->id); connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; @@ -855,8 +861,8 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta int nDoS = 0; if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second.first)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + // Don't send reject message with code 0 or an internal reject code. + if (it != mapBlockSource.end() && State(it->second.first) && state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) { CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; State(it->second.first)->rejects.push_back(reject); if (nDoS > 0 && it->second.second) @@ -987,6 +993,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + std::shared_ptr<const CBlock> a_recent_block; + std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; + bool fWitnessesPresentInARecentCompactBlock; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + a_recent_compact_block = most_recent_compact_block; + fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; + } if (mi != mapBlockIndex.end()) { if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && @@ -996,11 +1011,6 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // before ActivateBestChain but after AcceptBlock). // In this case, we need to run ActivateBestChain prior to checking the relay // conditions below. - std::shared_ptr<const CBlock> a_recent_block; - { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - } CValidationState dummy; ActivateBestChain(dummy, Params(), a_recent_block); } @@ -1024,7 +1034,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical if (send && connman.OutboundTargetReached(true) && ( ((pindexBestHeader != NULL) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) { - LogPrint("net", "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); //disconnect node pfrom->fDisconnect = true; @@ -1034,14 +1044,20 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // it's available before trying to send. if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { - // Send block from disk - CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) - assert(!"cannot load block from disk"); + std::shared_ptr<const CBlock> pblock; + if (a_recent_block && a_recent_block->GetHash() == (*mi).second->GetBlockHash()) { + pblock = a_recent_block; + } else { + // Send block from disk + std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); + if (!ReadBlockFromDisk(*pblockRead, (*mi).second, consensusParams)) + assert(!"cannot load block from disk"); + pblock = pblockRead; + } if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1050,7 +1066,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam LOCK(pfrom->cs_filter); if (pfrom->pfilter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(block, *pfrom->pfilter); + merkleBlock = CMerkleBlock(*pblock, *pfrom->pfilter); } } if (sendMerkleBlock) { @@ -1063,7 +1079,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1077,10 +1093,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { - CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - } else - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); + if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + } else { + CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + } + } else { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + } } // Trigger the peer node to send a getblocks request for the next batch of inventory @@ -1168,7 +1189,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic<bool>& interruptMsgProc) { - LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); if (IsArgSet("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1192,7 +1213,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (strCommand == NetMsgType::REJECT) { - if (fDebug) { + if (LogAcceptCategory(BCLog::NET)) { try { std::string strMsg; unsigned char ccode; std::string strReason; vRecv >> LIMITED_STRING(strMsg, CMessageHeader::COMMAND_SIZE) >> ccode >> LIMITED_STRING(strReason, MAX_REJECT_MESSAGE_LENGTH); @@ -1206,10 +1227,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr vRecv >> hash; ss << ": hash " << hash.ToString(); } - LogPrint("net", "Reject %s\n", SanitizeString(ss.str())); + LogPrint(BCLog::NET, "Reject %s\n", SanitizeString(ss.str())); } catch (const std::ios_base::failure&) { // Avoid feedback loops by preventing reject messages from triggering a new reject message. - LogPrint("net", "Unparseable reject message received\n"); + LogPrint(BCLog::NET, "Unparseable reject message received\n"); } } } @@ -1247,7 +1268,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nServicesExpected & ~nServices) { - LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); pfrom->fDisconnect = true; @@ -1335,11 +1356,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr FastRandomContext insecure_rand; if (addr.IsRoutable()) { - LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); + LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); pfrom->PushAddress(addr, insecure_rand); } else if (IsPeerAddrLocalGood(pfrom)) { addr.SetIP(addrMe); - LogPrint("net", "ProcessMessages: advertising address %s\n", addr.ToString()); + LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); pfrom->PushAddress(addr, insecure_rand); } } @@ -1541,7 +1562,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; bool fAlreadyHave = AlreadyHave(inv); - LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); if (inv.type == MSG_TX) { inv.type |= nFetchFlags; @@ -1556,16 +1577,17 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); - LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); } } else { pfrom->AddInventoryKnown(inv); - if (fBlocksOnly) - LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); - else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) + if (fBlocksOnly) { + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { pfrom->AskFor(inv); + } } // Track requests for our stuff @@ -1588,11 +1610,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return error("message getdata size() = %u", vInv.size()); } - if (fDebug || (vInv.size() != 1)) - LogPrint("net", "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); + LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); - if ((fDebug && vInv.size() > 0) || (vInv.size() == 1)) - LogPrint("net", "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + if (vInv.size() > 0) { + LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + } pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); ProcessGetData(pfrom, chainparams.GetConsensus(), connman, interruptMsgProc); @@ -1631,12 +1653,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); + LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) { - LogPrint("net", " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogPrint(BCLog::NET, " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } // If pruning, don't inv blocks unless we have on disk and are likely to still have @@ -1644,7 +1666,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= chainActive.Tip()->nHeight - nPrunedBlocksLikelyToHave)) { - LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); @@ -1652,7 +1674,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { // When this block is requested, we'll send an inv that'll // trigger the peer to getblocks the next batch of inventory. - LogPrint("net", " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + LogPrint(BCLog::NET, " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); pfrom->hashContinue = pindex->GetBlockHash(); break; } @@ -1693,7 +1715,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // might maliciously send lots of getblocktxn requests to trigger // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. - LogPrint("net", "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); CInv inv; inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; @@ -1718,7 +1740,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { - LogPrint("net", "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); return true; } @@ -1743,7 +1765,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; int nLimit = MAX_HEADERS_RESULTS; - LogPrint("net", "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -1773,7 +1795,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { - LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->id); return true; } @@ -1805,7 +1827,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->nLastTXTime = GetTime(); - LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", + LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom->id, tx.GetHash().ToString(), mempool.size(), mempool.DynamicMemoryUsage() / 1000); @@ -1835,7 +1857,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (setMisbehaving.count(fromPeer)) continue; if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2, &lRemovedTxn)) { - LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanTx, connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { vWorkQueue.emplace_back(orphanHash, i); @@ -1850,11 +1872,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Punish peer that gave us an invalid orphan tx Misbehaving(fromPeer, nDos); setMisbehaving.insert(fromPeer); - LogPrint("mempool", " invalid orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); } // Has inputs but not accepted to mempool // Probably non-standard or insufficient fee - LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); + LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); vEraseQueue.push_back(orphanHash); if (!orphanTx.HasWitness() && !stateDummy.CorruptionPossible()) { // Do not use rejection cache for witness transactions or @@ -1892,10 +1914,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // DoS prevention: do not allow mapOrphanTransactions to grow unbounded unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + if (nEvicted > 0) { + LogPrint(BCLog::MEMPOOL, "mapOrphan overflow, removed %u tx\n", nEvicted); + } } else { - LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); // We will continue to reject this tx since it has rejected // parents so avoid re-requesting it from other peers. recentRejects->insert(tx.GetHash()); @@ -1939,10 +1962,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr int nDoS = 0; if (state.IsInvalid(nDoS)) { - LogPrint("mempoolrej", "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), + LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); - if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P + if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); if (nDoS > 0) { @@ -2046,7 +2069,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); else { // The block was already in flight using compact blocks from the same peer - LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + LogPrint(BCLog::NET, "Peer sent us compact block we were already syncing!\n"); return true; } } @@ -2161,7 +2184,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || it->second.first != pfrom->GetId()) { - LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); return true; } @@ -2254,7 +2277,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), pindexBestHeader->nHeight, @@ -2296,7 +2319,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); CNodeState *nodestate = State(pfrom->GetId()); if (nodestate->nUnconnectingHeaders > 0) { - LogPrint("net", "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); } nodestate->nUnconnectingHeaders = 0; @@ -2307,7 +2330,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. - LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } @@ -2332,7 +2355,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // the main chain -- this shouldn't really happen. Bail out on the // direct fetch and rely on parallel download instead. if (!chainActive.Contains(pindexWalk)) { - LogPrint("net", "Large reorg, won't direct fetch to %s (%d)\n", + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } else { @@ -2346,11 +2369,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); - LogPrint("net", "Requesting block %s from peer=%d\n", + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", pindex->GetBlockHash().ToString(), pfrom->id); } if (vGetData.size() > 1) { - LogPrint("net", "Downloading blocks toward %s (%d) via headers direct fetch\n", + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { @@ -2370,7 +2393,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; - LogPrint("net", "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. @@ -2402,14 +2425,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. if (!pfrom->fInbound) { - LogPrint("net", "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); return true; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (pfrom->fSentAddr) { - LogPrint("net", "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); return true; } pfrom->fSentAddr = true; @@ -2426,14 +2449,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->fWhitelisted) { - LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; return true; } if (connman.OutboundTargetReached(false) && !pfrom->fWhitelisted) { - LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); + LogPrint(BCLog::NET, "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; return true; } @@ -2509,7 +2532,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (!(sProblem.empty())) { - LogPrint("net", "pong peer=%d: %s, %x expected, %x received, %u bytes\n", + LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", pfrom->id, sProblem, pfrom->nPingNonceSent, @@ -2587,7 +2610,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(pfrom->cs_feeFilter); pfrom->minFeeFilter = newFeeFilter; } - LogPrint("net", "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); } } @@ -2598,7 +2621,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { // Ignore unknown commands for extensibility - LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); } @@ -2876,7 +2899,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -2960,7 +2983,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { // We only send up to 1 block as header-and-ids, as otherwise // probably means we're doing an initial-ish-sync or they're slow - LogPrint("net", "%s sending header-and-ids %s to peer=%d\n", __func__, + LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; @@ -2969,7 +2992,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - if (state.fWantsCmpctWitness) + if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); else { CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); @@ -2988,12 +3011,12 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { if (vHeaders.size() > 1) { - LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + LogPrint(BCLog::NET, "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, vHeaders.size(), vHeaders.front().GetHash().ToString(), vHeaders.back().GetHash().ToString(), pto->id); } else { - LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, + LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->id); } connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); @@ -3015,14 +3038,14 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // This should be very rare and could be optimized out. // Just log for now. if (chainActive[pindex->nHeight] != pindex) { - LogPrint("net", "Announcing block %s not on main chain (tip=%s)\n", + LogPrint(BCLog::NET, "Announcing block %s not on main chain (tip=%s)\n", hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); } // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); - LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, + LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, pto->id, hashToAnnounce.ToString()); } } @@ -3201,13 +3224,13 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); - LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), pindex->nHeight, pto->id); } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { State(staller)->nStallingSince = nNow; - LogPrint("net", "Stall started peer=%d\n", staller); + LogPrint(BCLog::NET, "Stall started peer=%d\n", staller); } } } @@ -3220,8 +3243,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) { - if (fDebug) - LogPrint("net", "Requesting %s peer=%d\n", inv.ToString(), pto->id); + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->id); vGetData.push_back(inv); if (vGetData.size() >= 1000) { diff --git a/src/net_processing.h b/src/net_processing.h index 9e3f1b7156..f460595bc1 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -30,10 +30,10 @@ private: public: PeerLogicValidation(CConnman* connmanIn); - virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock); - virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); - virtual void BlockChecked(const CBlock& block, const CValidationState& state); - virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock); + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; + void BlockChecked(const CBlock& block, const CValidationState& state) override; + void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; }; struct CNodeStateStats { diff --git a/src/netbase.cpp b/src/netbase.cpp index 0f02e93e46..bdc725359a 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -281,7 +281,7 @@ std::string Socks5ErrorString(int err) static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, SOCKET& hSocket) { IntrRecvError recvr; - LogPrint("net", "SOCKS5 connecting %s\n", strDest); + LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); if (strDest.size() > 255) { CloseSocket(hSocket); return error("Hostname too long"); @@ -327,7 +327,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials CloseSocket(hSocket); return error("Error sending authentication to proxy"); } - LogPrint("proxy", "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); char pchRetA[2]; if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { CloseSocket(hSocket); @@ -409,7 +409,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials CloseSocket(hSocket); return error("Error reading from proxy"); } - LogPrint("net", "SOCKS5 connected %s\n", strDest); + LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } @@ -458,7 +458,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe int nRet = select(hSocket + 1, NULL, &fdset, NULL, &timeout); if (nRet == 0) { - LogPrint("net", "connection to %s timeout\n", addrConnect.ToString()); + LogPrint(BCLog::NET, "connection to %s timeout\n", addrConnect.ToString()); CloseSocket(hSocket); return false; } diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index da33e4100f..f3f7f8378e 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -7,14 +7,121 @@ #include "policy/policy.h" #include "amount.h" +#include "clientversion.h" #include "primitives/transaction.h" #include "random.h" #include "streams.h" #include "txmempool.h" #include "util.h" -void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay) +/** + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> txCtAvg; + // and calculate the total for the current block to update the moving average + std::vector<int> curBlockTxCt; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector<std::vector<double> > confAvg; // confAvg[Y][X] + // and calculate the totals for the current block to update the moving averages + std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] + + // Sum the total feerate of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> avg; + // and calculate the total for the current block to update the moving average + std::vector<double> curBlockVal; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg feerate per bucket + + double decay; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after MAX_CONFIRMS for each bucket + std::vector<int> oldUnconfTxs; + +public: + /** + * Create new TxConfirmStats. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundaries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + */ + TxConfirmStats(const std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); + + /** Clear the state of the curBlock variables to start counting for the new block */ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val the feerate of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight) const; + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() const { return confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout) const; + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein); +}; + + +TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, + unsigned int maxConfirms, double _decay) { decay = _decay; for (unsigned int i = 0; i < defaultBuckets.size(); i++) { @@ -77,7 +184,7 @@ void TxConfirmStats::UpdateMovingAverages() // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) + unsigned int nBlockHeight) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget @@ -165,7 +272,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, } } - LogPrint("estimatefee", "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", + LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", confTarget, requireGreater ? ">" : "<", successBreakPoint, requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); @@ -173,7 +280,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, return median; } -void TxConfirmStats::Write(CAutoFile& fileout) +void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; fileout << buckets; @@ -241,7 +348,7 @@ void TxConfirmStats::Read(CAutoFile& filein) for (unsigned int i = 0; i < buckets.size(); i++) bucketMap[buckets[i]] = i; - LogPrint("estimatefee", "Reading estimates: %u buckets counting confirms up to %u blocks\n", + LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); } @@ -260,24 +367,26 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet blocksAgo = 0; if (blocksAgo < 0) { - LogPrint("estimatefee", "Blockpolicy error, blocks ago is negative for mempool tx\n"); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n"); return; //This can't happen because we call this with our best seen height, no entries can have higher } if (blocksAgo >= (int)unconfTxs.size()) { - if (oldUnconfTxs[bucketindex] > 0) + if (oldUnconfTxs[bucketindex] > 0) { oldUnconfTxs[bucketindex]--; - else - LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n", + } else { + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n", bucketindex); + } } else { unsigned int blockIndex = entryHeight % unconfTxs.size(); - if (unconfTxs[blockIndex][bucketindex] > 0) + if (unconfTxs[blockIndex][bucketindex] > 0) { unconfTxs[blockIndex][bucketindex]--; - else - LogPrint("estimatefee", "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n", + } else { + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n", blockIndex, bucketindex); + } } } @@ -288,9 +397,10 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // of no harm to try to remove them again. bool CBlockPolicyEstimator::removeTx(uint256 hash) { + LOCK(cs_feeEstimator); std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); mapMemPoolTxs.erase(hash); return true; } else { @@ -308,15 +418,21 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() vfeelist.push_back(bucketBoundary); } vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + feeStats = new TxConfirmStats(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); +} + +CBlockPolicyEstimator::~CBlockPolicyEstimator() +{ + delete feeStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) { + LOCK(cs_feeEstimator); unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { - LogPrint("estimatefee", "Blockpolicy error mempool tx %s already being tracked\n", + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString().c_str()); return; } @@ -341,7 +457,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) @@ -358,20 +474,21 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM if (blocksToConfirm <= 0) { // This can't happen because we don't process transactions from a block with a height // lower than our greatest seen height - LogPrint("estimatefee", "Blockpolicy error Transaction had negative blocksToConfirm\n"); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n"); return false; } // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries) { + LOCK(cs_feeEstimator); if (nBlockHeight <= nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random // they don't affect the estimate. @@ -387,7 +504,7 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, nBestSeenHeight = nBlockHeight; // Clear the current block state and update unconfirmed circular buffer - feeStats.ClearCurrent(nBlockHeight); + feeStats->ClearCurrent(nBlockHeight); unsigned int countedTxs = 0; // Repopulate the current block states @@ -397,23 +514,24 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, } // Update all exponential averages with the current block state - feeStats.UpdateMovingAverages(); + feeStats->UpdateMovingAverages(); - LogPrint("estimatefee", "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); trackedTxs = 0; untrackedTxs = 0; } -CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { + LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + if (confTarget <= 1 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) return CFeeRate(0); - double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + double median = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); if (median < 0) return CFeeRate(0); @@ -421,22 +539,28 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) - return CFeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; double median = -1; - while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { - median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } + + { + LOCK(cs_feeEstimator); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + return CFeeRate(0); + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + while (median < 0 && (unsigned int)confTarget <= feeStats->GetMaxConfirms()) { + median = feeStats->EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + } + } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) *answerFoundAtTarget = confTarget - 1; @@ -452,19 +576,40 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } -void CBlockPolicyEstimator::Write(CAutoFile& fileout) +bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { - fileout << nBestSeenHeight; - feeStats.Write(fileout); + try { + LOCK(cs_feeEstimator); + fileout << 139900; // version required to read: 0.13.99 or later + fileout << CLIENT_VERSION; // version that wrote the file + fileout << nBestSeenHeight; + feeStats->Write(fileout); + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Write(): unable to read policy estimator data (non-fatal)\n"); + return false; + } + return true; } -void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) +bool CBlockPolicyEstimator::Read(CAutoFile& filein) { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - feeStats.Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + try { + LOCK(cs_feeEstimator); + int nVersionRequired, nVersionThatWrote, nFileBestSeenHeight; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + filein >> nFileBestSeenHeight; + feeStats->Read(filein); + nBestSeenHeight = nFileBestSeenHeight; + // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal)\n"); + return false; + } + return true; } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index dd01c90c45..34f07c7270 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -8,6 +8,7 @@ #include "amount.h" #include "uint256.h" #include "random.h" +#include "sync.h" #include <map> #include <string> @@ -17,6 +18,7 @@ class CAutoFile; class CFeeRate; class CTxMemPoolEntry; class CTxMemPool; +class TxConfirmStats; /** \class CBlockPolicyEstimator * The BlockPolicyEstimator is used for estimating the feerate needed @@ -59,113 +61,6 @@ class CTxMemPool; * they've been outstanding. */ -/** - * We will instantiate an instance of this class to track transactions that were - * included in a block. We will lump transactions into a bucket according to their - * approximate feerate and then track how long it took for those txs to be included in a block - * - * The tracking of unconfirmed (mempool) transactions is completely independent of the - * historical tracking of transactions that have been confirmed in a block. - */ -class TxConfirmStats -{ -private: - //Define the buckets we will group transactions into - std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) - std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket - - // For each bucket X: - // Count the total # of txs in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector<int> curBlockTxCt; - - // Count the total # of txs confirmed within Y blocks in each bucket - // Track the historical moving average of theses totals over blocks - std::vector<std::vector<double> > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] - - // Sum the total feerate of all tx's in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> avg; - // and calculate the total for the current block to update the moving average - std::vector<double> curBlockVal; - - // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg feerate per bucket - - double decay; - - // Mempool counts of outstanding transactions - // For each bucket X, track the number of transactions in the mempool - // that are unconfirmed for each possible confirmation value Y - std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket - std::vector<int> oldUnconfTxs; - -public: - /** - * Initialize the data structures. This is called by BlockPolicyEstimator's - * constructor with default values. - * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track - * @param decay how much to decay the historical moving average per block - */ - void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); - - /** Clear the state of the curBlock variables to start counting for the new block */ - void ClearCurrent(unsigned int nBlockHeight); - - /** - * Record a new transaction data point in the current block stats - * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val the feerate of the transaction - * @warning blocksToConfirm is 1-based and has to be >= 1 - */ - void Record(int blocksToConfirm, double val); - - /** Record a new transaction entering the mempool*/ - unsigned int NewTx(unsigned int nBlockHeight, double val); - - /** Remove a transaction from mempool tracking stats*/ - void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); - - /** Update our estimates by decaying our historical moving average and updating - with the data gathered from the current block */ - void UpdateMovingAverages(); - - /** - * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets - * to make sure we have enough data points) whose transactions still have sufficient likelihood - * of being confirmed within the target number of confirmations - * @param confTarget target number of confirmations - * @param sufficientTxVal required average number of transactions per block in a bucket range - * @param minSuccess the success probability we require - * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR - * return the highest feerate such that all lower values fail minSuccess - * @param nBlockHeight the current block height - */ - double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight); - - /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() { return confAvg.size(); } - - /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout); - - /** - * Read saved state of estimation data from a file and replace all internal data structures and - * variables with this state. - */ - void Read(CAutoFile& filein); -}; - - - /** Track confirm delays up to 25 blocks, can't estimate beyond that */ static const unsigned int MAX_BLOCK_CONFIRMS = 25; @@ -204,14 +99,12 @@ class CBlockPolicyEstimator public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); + ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries); - /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); - /** Process a transaction accepted to the mempool*/ void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); @@ -219,19 +112,19 @@ public: bool removeTx(uint256 hash); /** Return a feerate estimate */ - CFeeRate estimateFee(int confTarget); + CFeeRate estimateFee(int confTarget) const; /** Estimate feerate needed to get be included in a block within * confTarget blocks. If no answer can be given at confTarget, return an * estimate at the lowest target where one can be given. */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const; /** Write estimation data to a file */ - void Write(CAutoFile& fileout); + bool Write(CAutoFile& fileout) const; /** Read estimation data from a file */ - void Read(CAutoFile& filein, int nFileVersion); + bool Read(CAutoFile& filein); private: CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main @@ -247,10 +140,16 @@ private: std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats; + TxConfirmStats* feeStats; unsigned int trackedTxs; unsigned int untrackedTxs; + + mutable CCriticalSection cs_feeEstimator; + + /** Process a transaction confirmed in a block*/ + bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + }; class FeeFilterRounder diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index ec398f6627..6f8b6c2953 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -8,6 +8,7 @@ #include "policy/policy.h" #include "validation.h" +#include "coins.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 662e8037ca..23ec3ab434 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -10,6 +10,7 @@ #include "chainparams.h" #include "clientmodel.h" +#include "fs.h" #include "guiconstants.h" #include "guiutil.h" #include "intro.h" @@ -38,7 +39,6 @@ #include <stdint.h> -#include <boost/filesystem/operations.hpp> #include <boost/thread.hpp> #include <QApplication> @@ -152,15 +152,21 @@ static void initTranslations(QTranslator &qtTranslatorBase, QTranslator &qtTrans #if QT_VERSION < 0x050000 void DebugMessageHandler(QtMsgType type, const char *msg) { - const char *category = (type == QtDebugMsg) ? "qt" : NULL; - LogPrint(category, "GUI: %s\n", msg); + if (type == QtDebugMsg) { + LogPrint(BCLog::QT, "GUI: %s\n", msg); + } else { + LogPrintf("GUI: %s\n", msg); + } } #else void DebugMessageHandler(QtMsgType type, const QMessageLogContext& context, const QString &msg) { Q_UNUSED(context); - const char *category = (type == QtDebugMsg) ? "qt" : NULL; - LogPrint(category, "GUI: %s\n", msg.toStdString()); + if (type == QtDebugMsg) { + LogPrint(BCLog::QT, "GUI: %s\n", msg.toStdString()); + } else { + LogPrintf("GUI: %s\n", msg.toStdString()); + } } #endif @@ -602,7 +608,7 @@ int main(int argc, char *argv[]) /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes - if (!boost::filesystem::is_directory(GetDataDir(false))) + if (!fs::is_directory(GetDataDir(false))) { QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QObject::tr("Error: Specified data directory \"%1\" does not exist.").arg(QString::fromStdString(GetArg("-datadir", "")))); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index be79a67990..5c26baef9e 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -478,6 +478,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); connect(_clientModel, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); + modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); setNumBlocks(_clientModel->getNumBlocks(), _clientModel->getLastBlockDate(), _clientModel->getVerificationProgress(NULL), false); connect(_clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); @@ -505,8 +506,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // initialize the disable state of the tray icon with the current value in the model. setTrayIconVisible(optionsModel->getHideTrayIcon()); } - - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); } else { // Disable possibility to show main window via action toggleHideAction->setEnabled(false); diff --git a/src/qt/callback.h b/src/qt/callback.h new file mode 100644 index 0000000000..a8b593a652 --- /dev/null +++ b/src/qt/callback.h @@ -0,0 +1,30 @@ +#ifndef BITCOIN_QT_CALLBACK_H +#define BITCOIN_QT_CALLBACK_H + +#include <QObject> + +class Callback : public QObject +{ + Q_OBJECT +public Q_SLOTS: + virtual void call() = 0; +}; + +template <typename F> +class FunctionCallback : public Callback +{ + F f; + +public: + FunctionCallback(F f_) : f(std::move(f_)) {} + ~FunctionCallback() override {} + void call() override { f(this); } +}; + +template <typename F> +FunctionCallback<F>* makeCallback(F f) +{ + return new FunctionCallback<F>(std::move(f)); +} + +#endif // BITCOIN_QT_CALLBACK_H diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb10e49422..de00eacdb9 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -9,6 +9,7 @@ #include "guiutil.h" #include "peertablemodel.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "clientversion.h" @@ -36,6 +37,8 @@ ClientModel::ClientModel(OptionsModel *_optionsModel, QObject *parent) : banTableModel(0), pollTimer(0) { + cachedBestHeaderHeight = -1; + cachedBestHeaderTime = -1; peerTableModel = new PeerTableModel(this); banTableModel = new BanTableModel(this); pollTimer = new QTimer(this); @@ -74,18 +77,28 @@ int ClientModel::getNumBlocks() const int ClientModel::getHeaderTipHeight() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->nHeight; + if (cachedBestHeaderHeight == -1) { + // make sure we initially populate the cache via a cs_main lock + // otherwise we need to wait for a tip update + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderHeight; } int64_t ClientModel::getHeaderTipTime() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->GetBlockTime(); + if (cachedBestHeaderTime == -1) { + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderTime; } quint64 ClientModel::getTotalBytesRecv() const @@ -283,6 +296,11 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CB int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (fHeader) { + // cache best headers time and height to reduce future cs_main locks + clientmodel->cachedBestHeaderHeight = pIndex->nHeight; + clientmodel->cachedBestHeaderTime = pIndex->GetBlockTime(); + } // if we are in-sync, update the UI regardless of last update time if (!initialSync || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { //pass a async signal to the UI thread diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 2c10e633b8..4c92e2144e 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -8,6 +8,8 @@ #include <QObject> #include <QDateTime> +#include <atomic> + class AddressTableModel; class BanTableModel; class OptionsModel; @@ -81,6 +83,10 @@ public: QString formatClientStartupTime() const; QString dataDir() const; + // caches for the best header + mutable std::atomic<int> cachedBestHeaderHeight; + mutable std::atomic<int64_t> cachedBestHeaderTime; + private: OptionsModel *optionsModel; PeerTableModel *peerTableModel; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 1d19c65753..38ad6e9aab 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -15,6 +15,7 @@ #include "wallet/coincontrol.h" #include "init.h" +#include "policy/fees.h" #include "policy/policy.h" #include "validation.h" // For mempool #include "wallet/wallet.h" @@ -512,7 +513,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) nPayFee = coinControl->nMinimumTotalFee; @@ -592,7 +593,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (payTxFee.GetFeePerK() > 0) dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), payTxFee.GetFeePerK()) / 1000; else { - dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), mempool.estimateSmartFee(nTxConfirmTarget).GetFeePerK()) / 1000; + dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(nTxConfirmTarget, NULL, ::mempool).GetFeePerK()) / 1000; } QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 0b29201872..0f1b3f4a73 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -692,17 +692,34 @@ <item> <layout class="QHBoxLayout" name="horizontalLayout_Buttons"> <item> - <widget class="QPushButton" name="resetButton"> - <property name="toolTip"> - <string>Reset all client options to default.</string> - </property> - <property name="text"> - <string>&Reset Options</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> + <layout class="QVBoxLayout" name="verticalLayout_Buttons"> + <item> + <widget class="QPushButton" name="openBitcoinConfButton"> + <property name="toolTip"> + <string>Open the %1 configuration file from the working directory.</string> + </property> + <property name="text"> + <string>Open Configuration File</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="resetButton"> + <property name="toolTip"> + <string>Reset all client options to default.</string> + </property> + <property name="text"> + <string>&Reset Options</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> </item> <item> <spacer name="horizontalSpacer_1"> @@ -756,27 +773,48 @@ </spacer> </item> <item> - <widget class="QPushButton" name="okButton"> - <property name="text"> - <string>&OK</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - <property name="default"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="cancelButton"> - <property name="text"> - <string>&Cancel</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> + <layout class="QVBoxLayout" name="verticalLayout_4"> + <item> + <spacer name="verticalSpacer"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>40</height> + </size> + </property> + </spacer> + </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <widget class="QPushButton" name="okButton"> + <property name="text"> + <string>&OK</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + <property name="default"> + <bool>true</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="cancelButton"> + <property name="text"> + <string>&Cancel</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> + </item> + </layout> </item> </layout> </item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index cc183908d4..52256ca5c4 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -1235,7 +1235,7 @@ <bool>false</bool> </property> <property name="default"> - <bool>true</bool> + <bool>false</bool> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index bb5b2d4347..3f3f9b9ccb 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -9,6 +9,7 @@ #include "qvalidatedlineedit.h" #include "walletmodel.h" +#include "fs.h" #include "primitives/transaction.h" #include "init.h" #include "policy/policy.h" @@ -35,9 +36,6 @@ #include "shlwapi.h" #endif -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> -#include <boost/filesystem/detail/utf8_codecvt_facet.hpp> #include <boost/scoped_array.hpp> #include <QAbstractItemView> @@ -65,7 +63,7 @@ #include <QFontDatabase> #endif -static boost::filesystem::detail::utf8_codecvt_facet utf8; +static fs::detail::utf8_codecvt_facet utf8; #if defined(Q_OS_MAC) extern double NSAppKitVersionNumber; @@ -410,13 +408,29 @@ bool isObscured(QWidget *w) void openDebugLogfile() { - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; + fs::path pathDebug = GetDataDir() / "debug.log"; /* Open debug.log with the associated application */ - if (boost::filesystem::exists(pathDebug)) + if (fs::exists(pathDebug)) QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathDebug))); } +bool openBitcoinConf() +{ + boost::filesystem::path pathConfig = GetConfigFile(BITCOIN_CONF_FILENAME); + + /* Create the file */ + boost::filesystem::ofstream configFile(pathConfig, std::ios_base::app); + + if (!configFile.good()) + return false; + + configFile.close(); + + /* Open bitcoin.conf with the associated application */ + return QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathConfig))); +} + void SubstituteFonts(const QString& language) { #if defined(Q_OS_MAC) @@ -597,7 +611,7 @@ TableViewLastColumnResizingFixer::TableViewLastColumnResizingFixer(QTableView* t } #ifdef WIN32 -boost::filesystem::path static StartupShortcutPath() +fs::path static StartupShortcutPath() { std::string chain = ChainNameFromCommandLine(); if (chain == CBaseChainParams::MAIN) @@ -610,13 +624,13 @@ boost::filesystem::path static StartupShortcutPath() bool GetStartOnSystemStartup() { // check for Bitcoin*.lnk - return boost::filesystem::exists(StartupShortcutPath()); + return fs::exists(StartupShortcutPath()); } bool SetStartOnSystemStartup(bool fAutoStart) { // If the shortcut exists already, remove it for updating - boost::filesystem::remove(StartupShortcutPath()); + fs::remove(StartupShortcutPath()); if (fAutoStart) { @@ -686,10 +700,8 @@ bool SetStartOnSystemStartup(bool fAutoStart) // Follow the Desktop Application Autostart Spec: // http://standards.freedesktop.org/autostart-spec/autostart-spec-latest.html -boost::filesystem::path static GetAutostartDir() +fs::path static GetAutostartDir() { - namespace fs = boost::filesystem; - char* pszConfigHome = getenv("XDG_CONFIG_HOME"); if (pszConfigHome) return fs::path(pszConfigHome) / "autostart"; char* pszHome = getenv("HOME"); @@ -697,7 +709,7 @@ boost::filesystem::path static GetAutostartDir() return fs::path(); } -boost::filesystem::path static GetAutostartFilePath() +fs::path static GetAutostartFilePath() { std::string chain = ChainNameFromCommandLine(); if (chain == CBaseChainParams::MAIN) @@ -707,7 +719,7 @@ boost::filesystem::path static GetAutostartFilePath() bool GetStartOnSystemStartup() { - boost::filesystem::ifstream optionFile(GetAutostartFilePath()); + fs::ifstream optionFile(GetAutostartFilePath()); if (!optionFile.good()) return false; // Scan through file for "Hidden=true": @@ -727,7 +739,7 @@ bool GetStartOnSystemStartup() bool SetStartOnSystemStartup(bool fAutoStart) { if (!fAutoStart) - boost::filesystem::remove(GetAutostartFilePath()); + fs::remove(GetAutostartFilePath()); else { char pszExePath[MAX_PATH+1]; @@ -735,9 +747,9 @@ bool SetStartOnSystemStartup(bool fAutoStart) if (readlink("/proc/self/exe", pszExePath, sizeof(pszExePath)-1) == -1) return false; - boost::filesystem::create_directories(GetAutostartDir()); + fs::create_directories(GetAutostartDir()); - boost::filesystem::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out|std::ios_base::trunc); + fs::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out|std::ios_base::trunc); if (!optionFile.good()) return false; std::string chain = ChainNameFromCommandLine(); @@ -843,14 +855,17 @@ void restoreWindowGeometry(const QString& strSetting, const QSize& defaultSize, QPoint pos = settings.value(strSetting + "Pos").toPoint(); QSize size = settings.value(strSetting + "Size", defaultSize).toSize(); - if (!pos.x() && !pos.y()) { - QRect screen = QApplication::desktop()->screenGeometry(); - pos.setX((screen.width() - size.width()) / 2); - pos.setY((screen.height() - size.height()) / 2); - } - parent->resize(size); parent->move(pos); + + if ((!pos.x() && !pos.y()) || (QApplication::desktop()->screenNumber(parent) == -1)) + { + QRect screen = QApplication::desktop()->screenGeometry(); + QPoint defaultPos((screen.width() - defaultSize.width()) / 2, + (screen.height() - defaultSize.height()) / 2); + parent->resize(defaultSize); + parent->move(defaultPos); + } } void setClipboard(const QString& str) @@ -859,12 +874,12 @@ void setClipboard(const QString& str) QApplication::clipboard()->setText(str, QClipboard::Selection); } -boost::filesystem::path qstringToBoostPath(const QString &path) +fs::path qstringToBoostPath(const QString &path) { - return boost::filesystem::path(path.toStdString(), utf8); + return fs::path(path.toStdString(), utf8); } -QString boostPathToQString(const boost::filesystem::path &path) +QString boostPathToQString(const fs::path &path) { return QString::fromStdString(path.string(utf8)); } diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 913aa5e24b..d6aa8c4ea6 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_GUIUTIL_H #include "amount.h" +#include "fs.h" #include <QEvent> #include <QHeaderView> @@ -16,8 +17,6 @@ #include <QTableView> #include <QLabel> -#include <boost/filesystem.hpp> - class QValidatedLineEdit; class SendCoinsRecipient; @@ -114,6 +113,9 @@ namespace GUIUtil // Open debug.log void openDebugLogfile(); + // Open the config file + bool openBitcoinConf(); + // Replace invalid default fonts with known good ones void SubstituteFonts(const QString& language); @@ -183,10 +185,10 @@ namespace GUIUtil void restoreWindowGeometry(const QString& strSetting, const QSize &defaultSizeIn, QWidget *parent); /* Convert QString to OS specific boost path through UTF-8 */ - boost::filesystem::path qstringToBoostPath(const QString &path); + fs::path qstringToBoostPath(const QString &path); /* Convert OS specific boost path to QString through UTF-8 */ - QString boostPathToQString(const boost::filesystem::path &path); + QString boostPathToQString(const fs::path &path); /* Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(int secs); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 4939648ff0..2460a59109 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -6,6 +6,7 @@ #include "config/bitcoin-config.h" #endif +#include "fs.h" #include "intro.h" #include "ui_intro.h" @@ -13,8 +14,6 @@ #include "util.h" -#include <boost/filesystem.hpp> - #include <QFileDialog> #include <QSettings> #include <QMessageBox> @@ -70,7 +69,6 @@ FreespaceChecker::FreespaceChecker(Intro *_intro) void FreespaceChecker::check() { - namespace fs = boost::filesystem; QString dataDirStr = intro->getPathToCheck(); fs::path dataDir = GUIUtil::qstringToBoostPath(dataDirStr); uint64_t freeBytesAvailable = 0; @@ -190,7 +188,6 @@ QString Intro::getDefaultDataDirectory() bool Intro::pickDataDirectory() { - namespace fs = boost::filesystem; QSettings settings; /* If data directory provided on command line, no need to look at settings or show a picking dialog */ diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 7ff00b1e9e..efb25aaf18 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -232,6 +232,18 @@ void OptionsDialog::on_resetButton_clicked() } } +void OptionsDialog::on_openBitcoinConfButton_clicked() +{ + /* explain the purpose of the config file */ + QMessageBox::information(this, tr("Configuration options"), + tr("The configuration file is used to specify advanced user options which override GUI settings. " + "Additionally, any command-line options will override this configuration file.")); + + /* show an error if there was some problem opening the file */ + if (!GUIUtil::openBitcoinConf()) + QMessageBox::critical(this, tr("Error"), tr("The configuration file could not be opened.")); +} + void OptionsDialog::on_okButton_clicked() { mapper->submit(); diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index d98c1dc193..f9f5823c05 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -47,6 +47,7 @@ private Q_SLOTS: /* set OK button state (enabled / disabled) */ void setOkButtonState(bool fState); void on_resetButton_clicked(); + void on_openBitcoinConfButton_clicked(); void on_okButton_clicked(); void on_cancelButton_clicked(); diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 974a71ddca..fff072fd4c 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -166,7 +166,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const switch(index.column()) { case NetNodeId: - return rec->nodeStats.nodeid; + return (qint64)rec->nodeStats.nodeid; case Address: return QString::fromStdString(rec->nodeStats.addrName); case Subversion: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 5167232d6a..bb8aa23de8 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1118,7 +1118,7 @@ void RPCConsole::disconnectSelectedNode() for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Find the node, disconnect it and clear the selected node if(g_connman->DisconnectNode(id)) clearSelectedNode(); @@ -1135,7 +1135,7 @@ void RPCConsole::banSelectedNode(int bantime) for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index ed7eab03f3..098cda6d32 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,6 +21,7 @@ #include "validation.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" +#include "policy/fees.h" #include "wallet/wallet.h" #include <QFontMetrics> @@ -114,7 +115,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); - ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); } @@ -175,6 +175,9 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); updateGlobalFeeVariables(); + // set default rbf checkbox state + ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() == 0) @@ -660,7 +663,7 @@ void SendCoinsDialog::updateSmartFeeLabel() int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; int estimateFoundAtBlocks = nBlocksToConfirm; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks, ::mempool); if (feeRate <= CFeeRate(0)) // not enough data => minfee { ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index a7b82117d8..dada689731 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -6,6 +6,7 @@ #include "chainparams.h" #include "consensus/validation.h" +#include "fs.h" #include "validation.h" #include "rpc/register.h" #include "rpc/server.h" @@ -17,8 +18,6 @@ #include <QDir> #include <QtGlobal> -#include <boost/filesystem.hpp> - static UniValue rpcNestedTest_rpc(const JSONRPCRequest& request) { if (request.fHelp) { @@ -156,5 +155,5 @@ void RPCNestedTests::rpcNestedTests() delete pblocktree; pblocktree = nullptr; - boost::filesystem::remove_all(boost::filesystem::path(path)); + fs::remove_all(fs::path(path)); } diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index d8bcfedb7c..cae18f41a5 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -31,6 +31,9 @@ Q_IMPORT_PLUGIN(qjpcodecs) Q_IMPORT_PLUGIN(qtwcodecs) Q_IMPORT_PLUGIN(qkrcodecs) #else +#if defined(QT_QPA_PLATFORM_MINIMAL) +Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin); +#endif #if defined(QT_QPA_PLATFORM_XCB) Q_IMPORT_PLUGIN(QXcbIntegrationPlugin); #elif defined(QT_QPA_PLATFORM_WINDOWS) @@ -53,6 +56,11 @@ int main(int argc, char *argv[]) bool fInvalid = false; + // Prefer the "minimal" platform for the test instead of the normal default + // platform ("xcb", "windows", or "cocoa") so tests can't unintentially + // interfere with any background GUIs and don't require extra resources. + setenv("QT_QPA_PLATFORM", "minimal", 0); + // Don't remove this, it's needed to access // QApplication:: and QCoreApplication:: in the tests QApplication app(argc, argv); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 36d93361dd..f794b6b382 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,6 +1,7 @@ #include "wallettests.h" #include "qt/bitcoinamountfield.h" +#include "qt/callback.h" #include "qt/optionsmodel.h" #include "qt/platformstyle.h" #include "qt/qvalidatedlineedit.h" @@ -22,9 +23,7 @@ namespace //! Press "Yes" button in modal send confirmation dialog. void ConfirmSend() { - QTimer* timer = new QTimer; - timer->setSingleShot(true); - QObject::connect(timer, &QTimer::timeout, []() { + QTimer::singleShot(0, makeCallback([](Callback* callback) { for (QWidget* widget : QApplication::topLevelWidgets()) { if (widget->inherits("SendConfirmationDialog")) { SendConfirmationDialog* dialog = qobject_cast<SendConfirmationDialog*>(widget); @@ -33,8 +32,8 @@ void ConfirmSend() button->click(); } } - }); - timer->start(0); + delete callback; + }), SLOT(call())); } //! Send coins to address and return txid. @@ -69,6 +68,18 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) } //! Simple qt wallet tests. +// +// Test widgets can be debugged interactively calling show() on them and +// manually running the event loop, e.g.: +// +// sendCoinsDialog.show(); +// QEventLoop().exec(); +// +// This also requires overriding the default minimal Qt platform: +// +// src/qt/test/test_bitcoin-qt -platform xcb # Linux +// src/qt/test/test_bitcoin-qt -platform windows # Windows +// src/qt/test/test_bitcoin-qt -platform cocoa # macOS void WalletTests::walletTests() { // Set up wallet and chain with 101 blocks (1 mature block for spending). diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 7e16cc9dd4..d81188895b 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -273,7 +273,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco // // Debug view // - if (fDebug) + if (logCategories != BCLog::NONE) { strHTML += "<hr><br>" + tr("Debug information") + "<br><br>"; BOOST_FOREACH(const CTxIn& txin, wtx.tx->vin) diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index a9d9b6887e..4bb260aa58 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -18,14 +18,8 @@ */ bool TransactionRecord::showTransaction(const CWalletTx &wtx) { - if (wtx.IsCoinBase()) - { - // Ensures we show generated coins / mined transactions at depth 1 - if (!wtx.IsInMainChain()) - { - return false; - } - } + // There are currently no cases where we hide transactions, but + // we may want to use this in the future for things like RBF. return true; } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ebcac53c25..a2a9271904 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -13,6 +13,7 @@ #include "transactiontablemodel.h" #include "base58.h" +#include "chain.h" #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman diff --git a/src/random.cpp b/src/random.cpp index c1e313cbde..6187f16290 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -91,7 +91,7 @@ static void RandAddSeedPerfmon() if (ret == ERROR_SUCCESS) { RAND_add(vData.data(), nSize, nSize / 100.0); memory_cleanse(vData.data(), nSize); - LogPrint("rand", "%s: %lu bytes\n", __func__, nSize); + LogPrint(BCLog::RAND, "%s: %lu bytes\n", __func__, nSize); } else { static bool warned = false; // Warn only once if (!warned) { diff --git a/src/rest.cpp b/src/rest.cpp index 54eefcafe3..9dcaf269d6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -9,6 +9,7 @@ #include "primitives/transaction.h" #include "validation.h" #include "httpserver.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" @@ -55,12 +56,9 @@ struct CCoin { } }; -extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); -extern UniValue mempoolInfoToJSON(); -extern UniValue mempoolToJSON(bool fVerbose = false); -extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); -extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); +/* Defined in rawtransaction.cpp */ +void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); +void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 96254a8cb9..01066d0eb2 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -3,6 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "rpc/blockchain.h" + #include "amount.h" #include "chain.h" #include "chainparams.h" @@ -42,13 +44,6 @@ static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); -/** - * Get the difficulty of the net wrt to the given block index, or the chain tip if - * not provided. - * - * @return A floating point number that is a multiple of the main net minimum - * difficulty (4295032833 hashes). - */ double GetDifficulty(const CBlockIndex* blockindex) { if (blockindex == NULL) @@ -106,7 +101,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails) { UniValue result(UniValue::VOBJ); result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); @@ -383,7 +378,7 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) info.push_back(Pair("depends", depends)); } -UniValue mempoolToJSON(bool fVerbose = false) +UniValue mempoolToJSON(bool fVerbose) { if (fVerbose) { @@ -860,7 +855,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) else if (height > chainHeight) throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height."); else if (height > chainHeight - MIN_BLOCKS_TO_KEEP) { - LogPrint("rpc", "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); + LogPrint(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); height = chainHeight - MIN_BLOCKS_TO_KEEP; } diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h new file mode 100644 index 0000000000..c021441b0a --- /dev/null +++ b/src/rpc/blockchain.h @@ -0,0 +1,40 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_BLOCKCHAIN_H +#define BITCOIN_RPC_BLOCKCHAIN_H + +class CBlock; +class CBlockIndex; +class CScript; +class CTransaction; +class uint256; +class UniValue; + +/** + * Get the difficulty of the net wrt to the given block index, or the chain tip if + * not provided. + * + * @return A floating point number that is a multiple of the main net minimum + * difficulty (4295032833 hashes). + */ +double GetDifficulty(const CBlockIndex* blockindex = nullptr); + +/** Callback for when block tip changed. */ +void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); + +/** Block description to JSON */ +UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); + +/** Mempool information to JSON */ +UniValue mempoolInfoToJSON(); + +/** Mempool to JSON */ +UniValue mempoolToJSON(bool fVerbose = false); + +/** Block header to JSON */ +UniValue blockheaderToJSON(const CBlockIndex* blockindex); + +#endif + diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 2cb250a198..941bdd9379 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -81,7 +81,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getblockheader", 1, "verbose" }, { "gettransaction", 1, "include_watchonly" }, { "getrawtransaction", 1, "verbose" }, - { "createrawtransaction", 0, "transactions" }, + { "createrawtransaction", 0, "inputs" }, { "createrawtransaction", 1, "outputs" }, { "createrawtransaction", 2, "locktime" }, { "signrawtransaction", 1, "prevtxs" }, @@ -113,6 +113,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempoolancestors", 1, "verbose" }, { "getmempooldescendants", 1, "verbose" }, { "bumpfee", 1, "options" }, + { "logging", 0, "include" }, + { "logging", 1, "exclude" }, + { "disconnectnode", 1, "nodeid" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 4db8ffaa7d..4ce52a6c7f 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -15,7 +15,9 @@ #include "validation.h" #include "miner.h" #include "net.h" +#include "policy/fees.h" #include "pow.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "txmempool.h" #include "util.h" @@ -26,7 +28,6 @@ #include <stdint.h> #include <boost/assign/list_of.hpp> -#include <boost/shared_ptr.hpp> #include <univalue.h> @@ -94,7 +95,7 @@ UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(request.params.size() > 0 ? request.params[0].get_int() : 120, request.params.size() > 1 ? request.params[1].get_int() : -1); } -UniValue generateBlocks(boost::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) +UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) { static const int nInnerLoopCount = 0x10000; int nHeightStart = 0; @@ -166,7 +167,7 @@ UniValue generate(const JSONRPCRequest& request) nMaxTries = request.params[1].get_int(); } - boost::shared_ptr<CReserveScript> coinbaseScript; + std::shared_ptr<CReserveScript> coinbaseScript; GetMainSignals().ScriptForMining(coinbaseScript); // If the keypool is exhausted, no script is returned at all. Catch this. @@ -207,7 +208,7 @@ UniValue generatetoaddress(const JSONRPCRequest& request) if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); - boost::shared_ptr<CReserveScript> coinbaseScript(new CReserveScript()); + std::shared_ptr<CReserveScript> coinbaseScript = std::make_shared<CReserveScript>(); coinbaseScript->reserveScript = GetScriptForDestination(address.Get()); return generateBlocks(coinbaseScript, nGenerate, nMaxTries, false); @@ -709,7 +710,7 @@ public: submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {} protected: - virtual void BlockChecked(const CBlock& block, const CValidationState& stateIn) { + void BlockChecked(const CBlock& block, const CValidationState& stateIn) override { if (block.GetHash() != hash) return; found = true; @@ -719,7 +720,7 @@ protected: UniValue submitblock(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { throw std::runtime_error( "submitblock \"hexdata\" ( \"jsonparametersobject\" )\n" "\nAttempts to submit new block to network.\n" @@ -737,11 +738,17 @@ UniValue submitblock(const JSONRPCRequest& request) + HelpExampleCli("submitblock", "\"mydata\"") + HelpExampleRpc("submitblock", "\"mydata\"") ); + } std::shared_ptr<CBlock> blockptr = std::make_shared<CBlock>(); CBlock& block = *blockptr; - if (!DecodeHexBlk(block, request.params[0].get_str())) + if (!DecodeHexBlk(block, request.params[0].get_str())) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); + } + + if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block does not start with a coinbase"); + } uint256 hash = block.GetHash(); bool fBlockPresent = false; @@ -750,10 +757,12 @@ UniValue submitblock(const JSONRPCRequest& request) BlockMap::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) { CBlockIndex *pindex = mi->second; - if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) + if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { return "duplicate"; - if (pindex->nStatus & BLOCK_FAILED_MASK) + } + if (pindex->nStatus & BLOCK_FAILED_MASK) { return "duplicate-invalid"; + } // Otherwise, we might only have the header - process the block before returning fBlockPresent = true; } @@ -771,14 +780,15 @@ UniValue submitblock(const JSONRPCRequest& request) RegisterValidationInterface(&sc); bool fAccepted = ProcessNewBlock(Params(), blockptr, true, NULL); UnregisterValidationInterface(&sc); - if (fBlockPresent) - { - if (fAccepted && !sc.found) + if (fBlockPresent) { + if (fAccepted && !sc.found) { return "duplicate-inconclusive"; + } return "duplicate"; } - if (!sc.found) + if (!sc.found) { return "inconclusive"; + } return BIP22ValidationResult(sc.state); } @@ -809,7 +819,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (nBlocks < 1) nBlocks = 1; - CFeeRate feeRate = mempool.estimateFee(nBlocks); + CFeeRate feeRate = ::feeEstimator.estimateFee(nBlocks); if (feeRate == CFeeRate(0)) return -1.0; @@ -847,7 +857,7 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocks, &answerFound); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index de1bbe62e5..1f973a0c18 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,11 +4,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "base58.h" +#include "chain.h" #include "clientversion.h" #include "init.h" #include "validation.h" +#include "httpserver.h" #include "net.h" #include "netbase.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "timedata.h" #include "util.h" @@ -554,6 +557,73 @@ UniValue getmemoryinfo(const JSONRPCRequest& request) } } +uint32_t getCategoryMask(UniValue cats) { + cats = cats.get_array(); + uint32_t mask = 0; + for (unsigned int i = 0; i < cats.size(); ++i) { + uint32_t flag = 0; + std::string cat = cats[i].get_str(); + if (!GetLogCategory(&flag, &cat)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); + } + mask |= flag; + } + return mask; +} + +UniValue logging(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) { + throw std::runtime_error( + "logging [include,...] <exclude>\n" + "Gets and sets the logging configuration.\n" + "When called without an argument, returns the list of categories that are currently being debug logged.\n" + "When called with arguments, adds or removes categories from debug logging.\n" + "The valid logging categories are: " + ListLogCategories() + "\n" + "libevent logging is configured on startup and cannot be modified by this RPC during runtime." + "Arguments:\n" + "1. \"include\" (array of strings) add debug logging for these categories.\n" + "2. \"exclude\" (array of strings) remove debug logging for these categories.\n" + "\nResult: <categories> (string): a list of the logging categories that are active.\n" + "\nExamples:\n" + + HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + + HelpExampleRpc("logging", "[\"all\"], \"[libevent]\"") + ); + } + + uint32_t originalLogCategories = logCategories; + if (request.params.size() > 0 && request.params[0].isArray()) { + logCategories |= getCategoryMask(request.params[0]); + } + + if (request.params.size() > 1 && request.params[1].isArray()) { + logCategories &= ~getCategoryMask(request.params[1]); + } + + // Update libevent logging if BCLog::LIBEVENT has changed. + // If the library version doesn't allow it, UpdateHTTPServerLogging() returns false, + // in which case we should clear the BCLog::LIBEVENT flag. + // Throw an error if the user has explicitly asked to change only the libevent + // flag and it failed. + uint32_t changedLogCategories = originalLogCategories ^ logCategories; + if (changedLogCategories & BCLog::LIBEVENT) { + if (!UpdateHTTPServerLogging(logCategories & BCLog::LIBEVENT)) { + logCategories &= ~BCLog::LIBEVENT; + if (changedLogCategories == BCLog::LIBEVENT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "libevent logging cannot be updated when using libevent before v2.1.1."); + } + } + } + + UniValue result(UniValue::VOBJ); + std::vector<CLogCategoryActive> vLogCatActive = ListActiveLogCategories(); + for (const auto& logCatActive : vLogCatActive) { + result.pushKV(logCatActive.category, logCatActive.active); + } + + return result; +} + UniValue echo(const JSONRPCRequest& request) { if (request.fHelp) @@ -580,7 +650,8 @@ static const CRPCCommand commands[] = /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, true, {"timestamp"}}, { "hidden", "echo", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "echojson", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "logging", &logging, true, {"include", "exclude"}}, }; void RegisterMiscRPCCommands(CRPCTable &t) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 44c6e6d308..cde5ae723b 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -234,23 +234,43 @@ UniValue addnode(const JSONRPCRequest& request) UniValue disconnectnode(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() == 0 || request.params.size() >= 3) throw std::runtime_error( - "disconnectnode \"node\" \n" - "\nImmediately disconnects from the specified node.\n" + "disconnectnode \"[address]\" [nodeid]\n" + "\nImmediately disconnects from the specified peer node.\n" + "\nStrictly one out of 'address' and 'nodeid' can be provided to identify the node.\n" + "\nTo disconnect by nodeid, either set 'address' to the empty string, or call using the named 'nodeid' argument only.\n" "\nArguments:\n" - "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" + "1. \"address\" (string, optional) The IP address/port of the node\n" + "2. \"nodeid\" (number, optional) The node ID (see getpeerinfo for node IDs)\n" "\nExamples:\n" + HelpExampleCli("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleCli("disconnectnode", "\"\" 1") + HelpExampleRpc("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleRpc("disconnectnode", "\"\", 1") ); if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - bool ret = g_connman->DisconnectNode(request.params[0].get_str()); - if (!ret) + bool success; + const UniValue &address_arg = request.params[0]; + const UniValue &id_arg = request.params.size() < 2 ? NullUniValue : request.params[1]; + + if (!address_arg.isNull() && id_arg.isNull()) { + /* handle disconnect-by-address */ + success = g_connman->DisconnectNode(address_arg.get_str()); + } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { + /* handle disconnect-by-id */ + NodeId nodeid = (NodeId) id_arg.get_int64(); + success = g_connman->DisconnectNode(nodeid); + } else { + throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); + } + + if (!success) { throw JSONRPCError(RPC_CLIENT_NODE_NOT_CONNECTED, "Node not found in connected nodes"); + } return NullUniValue; } @@ -607,7 +627,7 @@ static const CRPCCommand commands[] = { "network", "ping", &ping, true, {} }, { "network", "getpeerinfo", &getpeerinfo, true, {} }, { "network", "addnode", &addnode, true, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, true, {"node"} }, + { "network", "disconnectnode", &disconnectnode, true, {"address", "nodeid"} }, { "network", "getaddednodeinfo", &getaddednodeinfo, true, {"node"} }, { "network", "getnettotals", &getnettotals, true, {} }, { "network", "getnetworkinfo", &getnetworkinfo, true, {} }, diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index 2be1edb5a6..823a5775f6 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -66,9 +66,9 @@ static const std::string COOKIEAUTH_USER = "__cookie__"; /** Default name for auth cookie file */ static const std::string COOKIEAUTH_FILE = ".cookie"; -boost::filesystem::path GetAuthCookieFile() +fs::path GetAuthCookieFile() { - boost::filesystem::path path(GetArg("-rpccookiefile", COOKIEAUTH_FILE)); + fs::path path(GetArg("-rpccookiefile", COOKIEAUTH_FILE)); if (!path.is_complete()) path = GetDataDir() / path; return path; } @@ -84,7 +84,7 @@ bool GenerateAuthCookie(std::string *cookie_out) * these are set to 077 in init.cpp unless overridden with -sysperms. */ std::ofstream file; - boost::filesystem::path filepath = GetAuthCookieFile(); + fs::path filepath = GetAuthCookieFile(); file.open(filepath.string().c_str()); if (!file.is_open()) { LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath.string()); @@ -103,7 +103,7 @@ bool GetAuthCookie(std::string *cookie_out) { std::ifstream file; std::string cookie; - boost::filesystem::path filepath = GetAuthCookieFile(); + fs::path filepath = GetAuthCookieFile(); file.open(filepath.string().c_str()); if (!file.is_open()) return false; @@ -118,8 +118,8 @@ bool GetAuthCookie(std::string *cookie_out) void DeleteAuthCookie() { try { - boost::filesystem::remove(GetAuthCookieFile()); - } catch (const boost::filesystem::filesystem_error& e) { + fs::remove(GetAuthCookieFile()); + } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove random auth cookie file: %s\n", __func__, e.what()); } } diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index 85bc4db101..70f7092cfe 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -6,11 +6,12 @@ #ifndef BITCOIN_RPCPROTOCOL_H #define BITCOIN_RPCPROTOCOL_H +#include "fs.h" + #include <list> #include <map> #include <stdint.h> #include <string> -#include <boost/filesystem.hpp> #include <univalue.h> @@ -89,7 +90,7 @@ std::string JSONRPCReply(const UniValue& result, const UniValue& error, const Un UniValue JSONRPCError(int code, const std::string& message); /** Get name of RPC authentication cookie file */ -boost::filesystem::path GetAuthCookieFile(); +fs::path GetAuthCookieFile(); /** Generate a new RPC authentication cookie and write it to disk */ bool GenerateAuthCookie(std::string *cookie_out); /** Read the RPC authentication cookie from disk */ diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 696f1bd781..717e9d75f3 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -360,7 +360,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) "it is not stored in the wallet or transmitted to the network.\n" "\nArguments:\n" - "1. \"inputs\" (string, required) A json array of json objects\n" + "1. \"inputs\" (array, required) A json array of json objects\n" " [\n" " {\n" " \"txid\":\"id\", (string, required) The transaction id\n" @@ -369,7 +369,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) " } \n" " ,...\n" " ]\n" - "2. \"outputs\" (string, required) a json object with outputs\n" + "2. \"outputs\" (object, required) a json object with outputs\n" " {\n" " \"address\": x.xxx, (numeric or string, required) The key is the bitcoin address, the numeric value (can be string) is the " + CURRENCY_UNIT + " amount\n" " \"data\": \"hex\" (string, required) The key is \"data\", the value is hex encoded data\n" @@ -936,7 +936,7 @@ static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- { "rawtransactions", "getrawtransaction", &getrawtransaction, true, {"txid","verbose"} }, - { "rawtransactions", "createrawtransaction", &createrawtransaction, true, {"transactions","outputs","locktime"} }, + { "rawtransactions", "createrawtransaction", &createrawtransaction, true, {"inputs","outputs","locktime"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, true, {"hexstring"} }, { "rawtransactions", "decodescript", &decodescript, true, {"hexstring"} }, { "rawtransactions", "sendrawtransaction", &sendrawtransaction, false, {"hexstring","allowhighfees"} }, diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 9b0699afcc..5c493428d8 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -6,6 +6,7 @@ #include "rpc/server.h" #include "base58.h" +#include "fs.h" #include "init.h" #include "random.h" #include "sync.h" @@ -16,7 +17,6 @@ #include <univalue.h> #include <boost/bind.hpp> -#include <boost/filesystem.hpp> #include <boost/foreach.hpp> #include <boost/shared_ptr.hpp> #include <boost/signals2/signal.hpp> @@ -305,7 +305,7 @@ bool CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) bool StartRPC() { - LogPrint("rpc", "Starting RPC\n"); + LogPrint(BCLog::RPC, "Starting RPC\n"); fRPCRunning = true; g_rpcSignals.Started(); return true; @@ -313,15 +313,16 @@ bool StartRPC() void InterruptRPC() { - LogPrint("rpc", "Interrupting RPC\n"); + LogPrint(BCLog::RPC, "Interrupting RPC\n"); // Interrupt e.g. running longpolls fRPCRunning = false; } void StopRPC() { - LogPrint("rpc", "Stopping RPC\n"); + LogPrint(BCLog::RPC, "Stopping RPC\n"); deadlineTimers.clear(); + DeleteAuthCookie(); g_rpcSignals.Stopped(); } @@ -368,8 +369,7 @@ void JSONRPCRequest::parse(const UniValue& valRequest) if (!valMethod.isStr()) throw JSONRPCError(RPC_INVALID_REQUEST, "Method must be a string"); strMethod = valMethod.get_str(); - if (strMethod != "getblocktemplate") - LogPrint("rpc", "ThreadRPCServer method=%s\n", SanitizeString(strMethod)); + LogPrint(BCLog::RPC, "ThreadRPCServer method=%s\n", SanitizeString(strMethod)); // Parse params UniValue valParams = find_value(request, "params"); @@ -531,7 +531,7 @@ void RPCRunLater(const std::string& name, boost::function<void(void)> func, int6 if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); deadlineTimers.erase(name); - LogPrint("rpc", "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); + LogPrint(BCLog::RPC, "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } diff --git a/src/rpc/server.h b/src/rpc/server.h index de14c7ed3e..34a9d3c24c 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -191,7 +191,6 @@ extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKe extern CAmount AmountFromValue(const UniValue& value); extern UniValue ValueFromAmount(const CAmount& amount); -extern double GetDifficulty(const CBlockIndex* blockindex = NULL); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); @@ -199,7 +198,6 @@ bool StartRPC(); void InterruptRPC(); void StopRPC(); std::string JSONRPCExecBatch(const UniValue& vReq); -void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); // Retrieves any serialization flags requested in command line argument int RPCSerializationFlags(); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 861c1a0220..0c1cfa2718 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -23,7 +23,9 @@ CScheduler::~CScheduler() #if BOOST_VERSION < 105000 static boost::system_time toPosixTime(const boost::chrono::system_clock::time_point& t) { - return boost::posix_time::from_time_t(boost::chrono::system_clock::to_time_t(t)); + // Creating the posix_time using from_time_t loses sub-second precision. So rather than exporting the time_point to time_t, + // start with a posix_time at the epoch (0) and add the milliseconds that have passed since then. + return boost::posix_time::from_time_t(0) + boost::posix_time::milliseconds(boost::chrono::duration_cast<boost::chrono::milliseconds>(t.time_since_epoch()).count()); } #endif diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 6f47b725fb..7bb8d9941b 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -15,28 +15,6 @@ #include <boost/thread.hpp> namespace { - -/** - * We're hashing a nonce into the entries themselves, so we don't need extra - * blinding in the set hash computation. - * - * This may exhibit platform endian dependent behavior but because these are - * nonced hashes (random) and this state is only ever used locally it is safe. - * All that matters is local consistency. - */ -class SignatureCacheHasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin()+4*hash_select, 4); - return u; - } -}; - /** * Valid signature cache, to avoid doing expensive ECDSA signature checking * twice for every transaction (once when accepted into memory pool, and diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 60690583de..55cec4cc8d 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -19,6 +19,27 @@ static const int64_t MAX_MAX_SIG_CACHE_SIZE = 16384; class CPubKey; +/** + * We're hashing a nonce into the entries themselves, so we don't need extra + * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. + */ +class SignatureCacheHasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; + } +}; + class CachingTransactionSignatureChecker : public TransactionSignatureChecker { private: diff --git a/src/serialize.h b/src/serialize.h index e4d72d2348..e82ddf2c5a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -336,11 +336,18 @@ I ReadVarInt(Stream& is) I n = 0; while(true) { unsigned char chData = ser_readdata8(is); + if (n > (std::numeric_limits<I>::max() >> 7)) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n = (n << 7) | (chData & 0x7F); - if (chData & 0x80) + if (chData & 0x80) { + if (n == std::numeric_limits<I>::max()) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n++; - else + } else { return n; + } } } diff --git a/src/streams.h b/src/streams.h index 1387b9cf54..8dc5a19ead 100644 --- a/src/streams.h +++ b/src/streams.h @@ -248,7 +248,8 @@ public: void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -261,7 +262,8 @@ public: void insert(iterator it, const char* first, const char* last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -339,6 +341,8 @@ public: void read(char* pch, size_t nSize) { + if (nSize == 0) return; + // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 4d6d6d6210..8cae4e66e8 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include "cuckoocache.h" +#include "script/sigcache.h" #include "test/test_bitcoin.h" #include "random.h" #include <thread> @@ -36,20 +37,6 @@ void insecure_GetRandHash(uint256& t) *(ptr++) = insecure_rand.rand32(); } -/** Definition copied from /src/script/sigcache.cpp - */ -class uint256Hasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin() + 4 * hash_select, 4); - return u; - } -}; /* Test that no values not inserted into the cache are read out of it. @@ -59,8 +46,9 @@ public: BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { insecure_rand = FastRandomContext(true); - CuckooCache::cache<uint256, uint256Hasher> cc{}; - cc.setup_bytes(32 << 20); + CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; + size_t megabytes = 4; + cc.setup_bytes(megabytes << 20); uint256 v; for (int x = 0; x < 100000; ++x) { insecure_GetRandHash(v); @@ -135,9 +123,9 @@ BOOST_AUTO_TEST_CASE(cuckoocache_hit_rate_ok) * as a lower bound on performance. */ double HitRateThresh = 0.98; - size_t megabytes = 32; + size_t megabytes = 4; for (double load = 0.1; load < 2; load *= 2) { - double hits = test_cache<CuckooCache::cache<uint256, uint256Hasher>>(megabytes, load); + double hits = test_cache<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes, load); BOOST_CHECK(normalize_hit_rate(hits, load) > HitRateThresh); } } @@ -204,8 +192,8 @@ void test_cache_erase(size_t megabytes) BOOST_AUTO_TEST_CASE(cuckoocache_erase_ok) { - size_t megabytes = 32; - test_cache_erase<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + size_t megabytes = 4; + test_cache_erase<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } template <typename Cache> @@ -291,8 +279,8 @@ void test_cache_erase_parallel(size_t megabytes) } BOOST_AUTO_TEST_CASE(cuckoocache_erase_parallel_ok) { - size_t megabytes = 32; - test_cache_erase_parallel<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + size_t megabytes = 4; + test_cache_erase_parallel<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } @@ -342,13 +330,13 @@ void test_cache_generations() } }; - const uint32_t BLOCK_SIZE = 10000; + const uint32_t BLOCK_SIZE = 1000; // We expect window size 60 to perform reasonably given that each epoch // stores 45% of the cache size (~472k). const uint32_t WINDOW_SIZE = 60; const uint32_t POP_AMOUNT = (BLOCK_SIZE / WINDOW_SIZE) / 2; const double load = 10; - const size_t megabytes = 32; + const size_t megabytes = 4; const size_t bytes = megabytes * (1 << 20); const uint32_t n_insert = static_cast<uint32_t>(load * (bytes / sizeof(uint256))); @@ -388,7 +376,7 @@ void test_cache_generations() } BOOST_AUTO_TEST_CASE(cuckoocache_generations) { - test_cache_generations<CuckooCache::cache<uint256, uint256Hasher>>(); + test_cache_generations<CuckooCache::cache<uint256, SignatureCacheHasher>>(); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 9d55beb8ea..c9d9849ada 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -28,7 +28,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper) // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'k'; uint256 in = GetRandHash(); @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'i'; @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_iterator) // Perform tests both obfuscated and non-obfuscated. for (int i = 0; i < 2; i++) { bool obfuscate = (bool)i; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); // The two keys are intentionally chosen for ordering @@ -125,8 +125,8 @@ BOOST_AUTO_TEST_CASE(dbwrapper_iterator) // Test that we do not obfuscation if there is existing data. BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) { - // We're going to share this boost::filesystem::path between two wrappers - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + // We're going to share this fs::path between two wrappers + fs::path ph = fs::temp_directory_path() / fs::unique_path(); create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. @@ -167,8 +167,8 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) // Ensure that we start obfuscating during a reindex. BOOST_AUTO_TEST_CASE(existing_data_reindex) { - // We're going to share this boost::filesystem::path between two wrappers - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + // We're going to share this fs::path between two wrappers + fs::path ph = fs::temp_directory_path() / fs::unique_path(); create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. @@ -204,7 +204,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) BOOST_AUTO_TEST_CASE(iterator_ordering) { - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, false); for (int x=0x00; x<256; ++x) { uint8_t key = x; @@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) { char buf[10]; - boost::filesystem::path ph = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); + fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, false); for (int x=0x00; x<10; ++x) { for (int y = 0; y < 10; y++) { diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index bc2f49ef3f..ed6782ea34 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -16,7 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool; + CBlockPolicyEstimator feeEst; + CTxMemPool mpool(&feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); @@ -78,16 +79,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // At this point we should need to combine 5 buckets to get enough data points // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(2) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(3) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); + BOOST_CHECK(feeEst.estimateSmartFee(1, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); + BOOST_CHECK(feeEst.estimateSmartFee(3, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); + BOOST_CHECK(feeEst.estimateSmartFee(4, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); + BOOST_CHECK(feeEst.estimateSmartFee(8, &answerFound, mpool) == feeEst.estimateFee(8) && answerFound == 8); } } @@ -99,7 +100,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Second highest feerate has 100% chance of being included by 2 blocks, // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { - origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); if (i > 2) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } @@ -118,10 +119,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -141,8 +142,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int answerFound; for (int i = 1; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateSmartFee(i, &answerFound, mpool).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); } // Mine all those transactions @@ -157,9 +158,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, 265); block.clear(); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } // Mine 200 more blocks where everything is mined every block @@ -179,9 +180,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); block.clear(); } - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee @@ -190,8 +191,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateFee(i).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 48593f62d5..cb625bda11 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -7,6 +7,7 @@ #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "fs.h" #include "key.h" #include "validation.h" #include "miner.h" @@ -24,7 +25,6 @@ #include <memory> -#include <boost/filesystem.hpp> #include <boost/thread.hpp> uint256 insecure_rand_seed = GetRandHash(); @@ -60,7 +60,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha RegisterAllCoreRPCCommands(tableRPC); ClearDatadirCache(); pathTemp = GetTempPath() / strprintf("test_bitcoin_%lu_%i", (unsigned long)GetTime(), (int)(GetRand(100000))); - boost::filesystem::create_directories(pathTemp); + fs::create_directories(pathTemp); ForceSetArg("-datadir", pathTemp.string()); mempool.setSanityCheck(1.0); pblocktree = new CBlockTreeDB(1 << 20, true); @@ -92,7 +92,7 @@ TestingSetup::~TestingSetup() delete pcoinsTip; delete pcoinsdbview; delete pblocktree; - boost::filesystem::remove_all(pathTemp); + fs::remove_all(pathTemp); } TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index a593f136eb..60a86d8c48 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -6,12 +6,12 @@ #define BITCOIN_TEST_TEST_BITCOIN_H #include "chainparamsbase.h" +#include "fs.h" #include "key.h" #include "pubkey.h" #include "txdb.h" #include "txmempool.h" -#include <boost/filesystem.hpp> #include <boost/thread.hpp> /** Basic testing setup. @@ -30,7 +30,7 @@ struct BasicTestingSetup { class CConnman; struct TestingSetup: public BasicTestingSetup { CCoinsViewDB *pcoinsdbview; - boost::filesystem::path pathTemp; + fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; diff --git a/src/test/testutil.cpp b/src/test/testutil.cpp index e6d8622979..591d0bf302 100644 --- a/src/test/testutil.cpp +++ b/src/test/testutil.cpp @@ -8,8 +8,8 @@ #include <shlobj.h> #endif -#include <boost/filesystem.hpp> +#include "fs.h" -boost::filesystem::path GetTempPath() { - return boost::filesystem::temp_directory_path(); +fs::path GetTempPath() { + return fs::temp_directory_path(); } diff --git a/src/test/testutil.h b/src/test/testutil.h index 5875dc50e6..cbe784d640 100644 --- a/src/test/testutil.h +++ b/src/test/testutil.h @@ -8,8 +8,8 @@ #ifndef BITCOIN_TEST_TESTUTIL_H #define BITCOIN_TEST_TESTUTIL_H -#include <boost/filesystem/path.hpp> +#include "fs.h" -boost::filesystem::path GetTempPath(); +fs::path GetTempPath(); #endif // BITCOIN_TEST_TESTUTIL_H diff --git a/src/timedata.cpp b/src/timedata.cpp index 2ff6437c73..ec74912703 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -58,7 +58,7 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) // Add data static CMedianFilter<int64_t> vTimeOffsets(BITCOIN_TIMEDATA_MAX_SAMPLES, 0); vTimeOffsets.input(nOffsetSample); - LogPrint("net","added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60); + LogPrint(BCLog::NET,"added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60); // There is a known issue here (see issue #4521): // @@ -108,11 +108,14 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) } } } - - BOOST_FOREACH(int64_t n, vSorted) - LogPrint("net", "%+d ", n); - LogPrint("net", "| "); - - LogPrint("net", "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset/60); + + if (LogAcceptCategory(BCLog::NET)) { + BOOST_FOREACH(int64_t n, vSorted) { + LogPrint(BCLog::NET, "%+d ", n); + } + LogPrint(BCLog::NET, "| "); + + LogPrint(BCLog::NET, "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset/60); + } } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index d24020e51f..c1bd95b00f 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -163,7 +163,7 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) self->reply_handlers.front()(*self, self->message); self->reply_handlers.pop_front(); } else { - LogPrint("tor", "tor: Received unexpected sync reply %i\n", self->message.code); + LogPrint(BCLog::TOR, "tor: Received unexpected sync reply %i\n", self->message.code); } } self->message.Clear(); @@ -182,13 +182,14 @@ void TorControlConnection::eventcb(struct bufferevent *bev, short what, void *ct { TorControlConnection *self = (TorControlConnection*)ctx; if (what & BEV_EVENT_CONNECTED) { - LogPrint("tor", "tor: Successfully connected!\n"); + LogPrint(BCLog::TOR, "tor: Successfully connected!\n"); self->connected(*self); } else if (what & (BEV_EVENT_EOF|BEV_EVENT_ERROR)) { - if (what & BEV_EVENT_ERROR) - LogPrint("tor", "tor: Error connecting to Tor control socket\n"); - else - LogPrint("tor", "tor: End of stream\n"); + if (what & BEV_EVENT_ERROR) { + LogPrint(BCLog::TOR, "tor: Error connecting to Tor control socket\n"); + } else { + LogPrint(BCLog::TOR, "tor: End of stream\n"); + } self->Disconnect(); self->disconnected(*self); } @@ -313,9 +314,9 @@ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string * @param maxsize Puts a maximum size limit on the file that is read. If the file is larger than this, truncated data * (with len > maxsize) will be returned. */ -static std::pair<bool,std::string> ReadBinaryFile(const std::string &filename, size_t maxsize=std::numeric_limits<size_t>::max()) +static std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxsize=std::numeric_limits<size_t>::max()) { - FILE *f = fopen(filename.c_str(), "rb"); + FILE *f = fsbridge::fopen(filename, "rb"); if (f == NULL) return std::make_pair(false,""); std::string retval; @@ -333,9 +334,9 @@ static std::pair<bool,std::string> ReadBinaryFile(const std::string &filename, s /** Write contents of std::string to a file. * @return true on success. */ -static bool WriteBinaryFile(const std::string &filename, const std::string &data) +static bool WriteBinaryFile(const fs::path &filename, const std::string &data) { - FILE *f = fopen(filename.c_str(), "wb"); + FILE *f = fsbridge::fopen(filename, "wb"); if (f == NULL) return false; if (fwrite(data.data(), 1, data.size(), f) != data.size()) { @@ -358,7 +359,7 @@ public: ~TorController(); /** Get name fo file to store private key in */ - std::string GetPrivateKeyFile(); + fs::path GetPrivateKeyFile(); /** Reconnect, after getting disconnected */ void Reconnect(); @@ -410,7 +411,7 @@ TorController::TorController(struct event_base* _base, const std::string& _targe // Read service private key if cached std::pair<bool,std::string> pkf = ReadBinaryFile(GetPrivateKeyFile()); if (pkf.first) { - LogPrint("tor", "tor: Reading cached private key from %s\n", GetPrivateKeyFile()); + LogPrint(BCLog::TOR, "tor: Reading cached private key from %s\n", GetPrivateKeyFile().string()); private_key = pkf.second; } } @@ -429,7 +430,7 @@ TorController::~TorController() void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { - LogPrint("tor", "tor: ADD_ONION successful\n"); + LogPrint(BCLog::TOR, "tor: ADD_ONION successful\n"); BOOST_FOREACH(const std::string &s, reply.lines) { std::map<std::string,std::string> m = ParseTorReplyMapping(s); std::map<std::string,std::string>::iterator i; @@ -441,9 +442,9 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe service = LookupNumeric(std::string(service_id+".onion").c_str(), GetListenPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { - LogPrint("tor", "tor: Cached service private key to %s\n", GetPrivateKeyFile()); + LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", GetPrivateKeyFile().string()); } else { - LogPrintf("tor: Error writing service private key to %s\n", GetPrivateKeyFile()); + LogPrintf("tor: Error writing service private key to %s\n", GetPrivateKeyFile().string()); } AddLocal(service, LOCAL_MANUAL); // ... onion requested - keep connection open @@ -457,7 +458,7 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { - LogPrint("tor", "tor: Authentication successful\n"); + LogPrint(BCLog::TOR, "tor: Authentication successful\n"); // Now that we know Tor is running setup the proxy for onion addresses // if -onion isn't set to something else. @@ -511,13 +512,13 @@ static std::vector<uint8_t> ComputeResponse(const std::string &key, const std::v void TorController::authchallenge_cb(TorControlConnection& _conn, const TorControlReply& reply) { if (reply.code == 250) { - LogPrint("tor", "tor: SAFECOOKIE authentication challenge successful\n"); + LogPrint(BCLog::TOR, "tor: SAFECOOKIE authentication challenge successful\n"); std::pair<std::string,std::string> l = SplitTorReplyLine(reply.lines[0]); if (l.first == "AUTHCHALLENGE") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); std::vector<uint8_t> serverHash = ParseHex(m["SERVERHASH"]); std::vector<uint8_t> serverNonce = ParseHex(m["SERVERNONCE"]); - LogPrint("tor", "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); + LogPrint(BCLog::TOR, "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); if (serverNonce.size() != 32) { LogPrintf("tor: ServerNonce is not 32 bytes, as required by spec\n"); return; @@ -562,12 +563,12 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); std::map<std::string,std::string>::iterator i; if ((i = m.find("Tor")) != m.end()) { - LogPrint("tor", "tor: Connected to Tor version %s\n", i->second); + LogPrint(BCLog::TOR, "tor: Connected to Tor version %s\n", i->second); } } } BOOST_FOREACH(const std::string &s, methods) { - LogPrint("tor", "tor: Supported authentication method: %s\n", s); + LogPrint(BCLog::TOR, "tor: Supported authentication method: %s\n", s); } // Prefer NULL, otherwise SAFECOOKIE. If a password is provided, use HASHEDPASSWORD /* Authentication: @@ -577,18 +578,18 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro std::string torpassword = GetArg("-torpassword", ""); if (!torpassword.empty()) { if (methods.count("HASHEDPASSWORD")) { - LogPrint("tor", "tor: Using HASHEDPASSWORD authentication\n"); + LogPrint(BCLog::TOR, "tor: Using HASHEDPASSWORD authentication\n"); boost::replace_all(torpassword, "\"", "\\\""); _conn.Command("AUTHENTICATE \"" + torpassword + "\"", boost::bind(&TorController::auth_cb, this, _1, _2)); } else { LogPrintf("tor: Password provided with -torpassword, but HASHEDPASSWORD authentication is not available\n"); } } else if (methods.count("NULL")) { - LogPrint("tor", "tor: Using NULL authentication\n"); + LogPrint(BCLog::TOR, "tor: Using NULL authentication\n"); _conn.Command("AUTHENTICATE", boost::bind(&TorController::auth_cb, this, _1, _2)); } else if (methods.count("SAFECOOKIE")) { // Cookie: hexdump -e '32/1 "%02x""\n"' ~/.tor/control_auth_cookie - LogPrint("tor", "tor: Using SAFECOOKIE authentication, reading cookie authentication from %s\n", cookiefile); + LogPrint(BCLog::TOR, "tor: Using SAFECOOKIE authentication, reading cookie authentication from %s\n", cookiefile); std::pair<bool,std::string> status_cookie = ReadBinaryFile(cookiefile, TOR_COOKIE_SIZE); if (status_cookie.first && status_cookie.second.size() == TOR_COOKIE_SIZE) { // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), boost::bind(&TorController::auth_cb, this, _1, _2)); @@ -630,7 +631,7 @@ void TorController::disconnected_cb(TorControlConnection& _conn) if (!reconnect) return; - LogPrint("tor", "tor: Not connected to Tor control port %s, trying to reconnect\n", target); + LogPrint(BCLog::TOR, "tor: Not connected to Tor control port %s, trying to reconnect\n", target); // Single-shot timer for reconnect. Use exponential backoff. struct timeval time = MillisToTimeval(int64_t(reconnect_timeout * 1000.0)); @@ -650,9 +651,9 @@ void TorController::Reconnect() } } -std::string TorController::GetPrivateKeyFile() +fs::path TorController::GetPrivateKeyFile() { - return (GetDataDir() / "onion_private_key").string(); + return GetDataDir() / "onion_private_key"; } void TorController::reconnect_cb(evutil_socket_t fd, short what, void *arg) diff --git a/src/txdb.cpp b/src/txdb.cpp index 1a30bb58ad..a3889fdf79 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -63,7 +63,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); + LogPrint(BCLog::COINDB, "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); } diff --git a/src/txdb.h b/src/txdb.h index 7f5cf2b583..d9214ba618 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -21,8 +21,14 @@ class CBlockIndex; class CCoinsViewDBCursor; class uint256; +//! Compensate for extra memory peak (x1.5-x1.9) at flush time. +static constexpr int DB_PEAK_USAGE_FACTOR = 2; +//! No need to periodic flush if at least this much space still available. +static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; +//! Always periodic flush if less than this much space still available. +static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) -static const int64_t nDefaultDbCache = 300; +static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache (MiB) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 6e1d7a42b5..ac842da6bf 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,7 +5,6 @@ #include "txmempool.h" -#include "clientversion.h" #include "consensus/consensus.h" #include "consensus/validation.h" #include "validation.h" @@ -16,7 +15,6 @@ #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, @@ -333,8 +331,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool() : - nTransactionsUpdated(0) +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -342,13 +340,6 @@ CTxMemPool::CTxMemPool() : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(); -} - -CTxMemPool::~CTxMemPool() -{ - delete minerPolicyEstimator; } void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) @@ -427,7 +418,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -457,7 +448,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -591,7 +582,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -634,7 +625,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (GetRand(std::numeric_limits<uint32_t>::max()) >= nCheckFrequency) return; - LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); + LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; uint64_t innerUsage = 0; @@ -850,51 +841,6 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -CFeeRate CTxMemPool::estimateFee(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateFee(nBlocks); -} -CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); -} - -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const -{ - try { - LOCK(cs); - fileout << 139900; // version required to read: 0.13.99 or later - fileout << CLIENT_VERSION; // version that wrote the file - minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) -{ - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { @@ -912,6 +858,13 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD BOOST_FOREACH(txiter ancestorIt, setAncestors) { mapTx.modify(ancestorIt, update_descendant_state(0, nFeeDelta, 0)); } + // Now update all descendants' modified fees with ancestors + setEntries setDescendants; + CalculateDescendants(it, setDescendants); + setDescendants.erase(it); + BOOST_FOREACH(txiter descendantIt, setDescendants) { + mapTx.modify(descendantIt, update_ancestor_state(0, nFeeDelta, 0, 0)); + } } } LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); @@ -1108,8 +1061,9 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe } } - if (maxFeeRateRemoved > CFeeRate(0)) - LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + if (maxFeeRateRemoved > CFeeRate(0)) { + LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + } } bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLimit) const { diff --git a/src/txmempool.h b/src/txmempool.h index 4222789510..92c4d9f9d4 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -496,8 +496,7 @@ public: /** Create a new CTxMemPool. */ - CTxMemPool(); - ~CTxMemPool(); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -618,19 +617,6 @@ public: TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; - /** Estimate fee rate needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate fee rate needed to get into the next nBlocks */ - CFeeRate estimateFee(int nBlocks) const; - - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile& fileout) const; - bool ReadFeeEstimates(CAutoFile& filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal<void (CTransactionRef)> NotifyEntryAdded; diff --git a/src/util.cpp b/src/util.cpp index 486df772fb..0dc203cba5 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -10,6 +10,7 @@ #include "util.h" #include "chainparamsbase.h" +#include "fs.h" #include "random.h" #include "serialize.h" #include "sync.h" @@ -72,11 +73,13 @@ #include <sys/prctl.h> #endif +#ifdef HAVE_MALLOPT_ARENA_MAX +#include <malloc.h> +#endif + #include <boost/algorithm/string/case_conv.hpp> // for to_lower() #include <boost/algorithm/string/join.hpp> #include <boost/algorithm/string/predicate.hpp> // for startswith() and endswith() -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> #include <boost/foreach.hpp> #include <boost/program_options/detail/config_file.hpp> #include <boost/program_options/parsers.hpp> @@ -106,7 +109,6 @@ CCriticalSection cs_args; std::map<std::string, std::string> mapArgs; static std::map<std::string, std::vector<std::string> > _mapMultiArgs; const std::map<std::string, std::vector<std::string> >& mapMultiArgs = _mapMultiArgs; -bool fDebug = false; bool fPrintToConsole = false; bool fPrintToDebugLog = true; @@ -116,27 +118,28 @@ bool fLogIPs = DEFAULT_LOGIPS; std::atomic<bool> fReopenDebugLog(false); CTranslationInterface translationInterface; +/** Log categories bitfield. */ +std::atomic<uint32_t> logCategories(0); + /** Init OpenSSL library multithreading support */ -static CCriticalSection** ppmutexOpenSSL; +static std::unique_ptr<CCriticalSection[]> ppmutexOpenSSL; void locking_callback(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS { if (mode & CRYPTO_LOCK) { - ENTER_CRITICAL_SECTION(*ppmutexOpenSSL[i]); + ENTER_CRITICAL_SECTION(ppmutexOpenSSL[i]); } else { - LEAVE_CRITICAL_SECTION(*ppmutexOpenSSL[i]); + LEAVE_CRITICAL_SECTION(ppmutexOpenSSL[i]); } } -// Init +// Singleton for wrapping OpenSSL setup/teardown. class CInit { public: CInit() { // Init OpenSSL library multithreading support - ppmutexOpenSSL = (CCriticalSection**)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(CCriticalSection*)); - for (int i = 0; i < CRYPTO_num_locks(); i++) - ppmutexOpenSSL[i] = new CCriticalSection(); + ppmutexOpenSSL.reset(new CCriticalSection[CRYPTO_num_locks()]); CRYPTO_set_locking_callback(locking_callback); // OpenSSL can optionally load a config file which lists optional loadable modules and engines. @@ -160,9 +163,8 @@ public: RAND_cleanup(); // Shutdown OpenSSL library multithreading support CRYPTO_set_locking_callback(NULL); - for (int i = 0; i < CRYPTO_num_locks(); i++) - delete ppmutexOpenSSL[i]; - OPENSSL_free(ppmutexOpenSSL); + // Clear the set of locks now to maintain symmetry with the constructor. + ppmutexOpenSSL.reset(); } } instance_of_cinit; @@ -212,8 +214,8 @@ void OpenDebugLog() assert(fileout == NULL); assert(vMsgsBeforeOpenLog); - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; - fileout = fopen(pathDebug.string().c_str(), "a"); + fs::path pathDebug = GetDataDir() / "debug.log"; + fileout = fsbridge::fopen(pathDebug, "a"); if (fileout) { setbuf(fileout, NULL); // unbuffered // dump buffered messages from before we opened the log @@ -227,36 +229,85 @@ void OpenDebugLog() vMsgsBeforeOpenLog = NULL; } -bool LogAcceptCategory(const char* category) +struct CLogCategoryDesc +{ + uint32_t flag; + std::string category; +}; + +const CLogCategoryDesc LogCategories[] = +{ + {BCLog::NONE, "0"}, + {BCLog::NET, "net"}, + {BCLog::TOR, "tor"}, + {BCLog::MEMPOOL, "mempool"}, + {BCLog::HTTP, "http"}, + {BCLog::BENCH, "bench"}, + {BCLog::ZMQ, "zmq"}, + {BCLog::DB, "db"}, + {BCLog::RPC, "rpc"}, + {BCLog::ESTIMATEFEE, "estimatefee"}, + {BCLog::ADDRMAN, "addrman"}, + {BCLog::SELECTCOINS, "selectcoins"}, + {BCLog::REINDEX, "reindex"}, + {BCLog::CMPCTBLOCK, "cmpctblock"}, + {BCLog::RAND, "rand"}, + {BCLog::PRUNE, "prune"}, + {BCLog::PROXY, "proxy"}, + {BCLog::MEMPOOLREJ, "mempoolrej"}, + {BCLog::LIBEVENT, "libevent"}, + {BCLog::COINDB, "coindb"}, + {BCLog::QT, "qt"}, + {BCLog::LEVELDB, "leveldb"}, + {BCLog::ALL, "1"}, + {BCLog::ALL, "all"}, +}; + +bool GetLogCategory(uint32_t *f, const std::string *str) +{ + if (f && str) { + if (*str == "") { + *f = BCLog::ALL; + return true; + } + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + if (LogCategories[i].category == *str) { + *f = LogCategories[i].flag; + return true; + } + } + } + return false; +} + +std::string ListLogCategories() { - if (category != NULL) - { - if (!fDebug) - return false; - - // Give each thread quick access to -debug settings. - // This helps prevent issues debugging global destructors, - // where mapMultiArgs might be deleted before another - // global destructor calls LogPrint() - static boost::thread_specific_ptr<std::set<std::string> > ptrCategory; - if (ptrCategory.get() == NULL) - { - if (mapMultiArgs.count("-debug")) { - const std::vector<std::string>& categories = mapMultiArgs.at("-debug"); - ptrCategory.reset(new std::set<std::string>(categories.begin(), categories.end())); - // thread_specific_ptr automatically deletes the set when the thread ends. - } else - ptrCategory.reset(new std::set<std::string>()); + std::string ret; + int outcount = 0; + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + // Omit the special cases. + if (LogCategories[i].flag != BCLog::NONE && LogCategories[i].flag != BCLog::ALL) { + if (outcount != 0) ret += ", "; + ret += LogCategories[i].category; + outcount++; } - const std::set<std::string>& setCategories = *ptrCategory; + } + return ret; +} - // if not debugging everything and not debugging specific category, LogPrint does nothing. - if (setCategories.count(std::string("")) == 0 && - setCategories.count(std::string("1")) == 0 && - setCategories.count(std::string(category)) == 0) - return false; +std::vector<CLogCategoryActive> ListActiveLogCategories() +{ + std::vector<CLogCategoryActive> ret; + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + // Omit the special cases. + if (LogCategories[i].flag != BCLog::NONE && LogCategories[i].flag != BCLog::ALL) { + CLogCategoryActive catActive; + catActive.category = LogCategories[i].category; + catActive.active = LogAcceptCategory(LogCategories[i].flag); + ret.push_back(catActive); + } } - return true; + return ret; } /** @@ -317,8 +368,8 @@ int LogPrintStr(const std::string &str) // reopen the log file, if requested if (fReopenDebugLog) { fReopenDebugLog = false; - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; - if (freopen(pathDebug.string().c_str(),"a",fileout) != NULL) + fs::path pathDebug = GetDataDir() / "debug.log"; + if (fsbridge::freopen(pathDebug,"a",fileout) != NULL) setbuf(fileout, NULL); // unbuffered } @@ -475,9 +526,8 @@ void PrintExceptionContinue(const std::exception* pex, const char* pszThread) fprintf(stderr, "\n\n************************\n%s\n", message.c_str()); } -boost::filesystem::path GetDefaultDataDir() +fs::path GetDefaultDataDir() { - namespace fs = boost::filesystem; // Windows < Vista: C:\Documents and Settings\Username\Application Data\Bitcoin // Windows >= Vista: C:\Users\Username\AppData\Roaming\Bitcoin // Mac: ~/Library/Application Support/Bitcoin @@ -502,13 +552,12 @@ boost::filesystem::path GetDefaultDataDir() #endif } -static boost::filesystem::path pathCached; -static boost::filesystem::path pathCachedNetSpecific; +static fs::path pathCached; +static fs::path pathCachedNetSpecific; static CCriticalSection csPathCached; -const boost::filesystem::path &GetDataDir(bool fNetSpecific) +const fs::path &GetDataDir(bool fNetSpecific) { - namespace fs = boost::filesystem; LOCK(csPathCached); @@ -540,13 +589,13 @@ void ClearDatadirCache() { LOCK(csPathCached); - pathCached = boost::filesystem::path(); - pathCachedNetSpecific = boost::filesystem::path(); + pathCached = fs::path(); + pathCachedNetSpecific = fs::path(); } -boost::filesystem::path GetConfigFile(const std::string& confPath) +fs::path GetConfigFile(const std::string& confPath) { - boost::filesystem::path pathConfigFile(confPath); + fs::path pathConfigFile(confPath); if (!pathConfigFile.is_complete()) pathConfigFile = GetDataDir(false) / pathConfigFile; @@ -555,7 +604,7 @@ boost::filesystem::path GetConfigFile(const std::string& confPath) void ReadConfigFile(const std::string& confPath) { - boost::filesystem::ifstream streamConfig(GetConfigFile(confPath)); + fs::ifstream streamConfig(GetConfigFile(confPath)); if (!streamConfig.good()) return; // No bitcoin.conf file is OK @@ -580,16 +629,16 @@ void ReadConfigFile(const std::string& confPath) } #ifndef WIN32 -boost::filesystem::path GetPidFile() +fs::path GetPidFile() { - boost::filesystem::path pathPidFile(GetArg("-pid", BITCOIN_PID_FILENAME)); + fs::path pathPidFile(GetArg("-pid", BITCOIN_PID_FILENAME)); if (!pathPidFile.is_complete()) pathPidFile = GetDataDir() / pathPidFile; return pathPidFile; } -void CreatePidFile(const boost::filesystem::path &path, pid_t pid) +void CreatePidFile(const fs::path &path, pid_t pid) { - FILE* file = fopen(path.string().c_str(), "w"); + FILE* file = fsbridge::fopen(path, "w"); if (file) { fprintf(file, "%d\n", pid); @@ -598,7 +647,7 @@ void CreatePidFile(const boost::filesystem::path &path, pid_t pid) } #endif -bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest) +bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 return MoveFileExA(src.string().c_str(), dest.string().c_str(), @@ -614,13 +663,13 @@ bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest) * Specifically handles case where path p exists, but it wasn't possible for the user to * write to the parent directory. */ -bool TryCreateDirectory(const boost::filesystem::path& p) +bool TryCreateDirectory(const fs::path& p) { try { - return boost::filesystem::create_directory(p); - } catch (const boost::filesystem::filesystem_error&) { - if (!boost::filesystem::exists(p) || !boost::filesystem::is_directory(p)) + return fs::create_directory(p); + } catch (const fs::filesystem_error&) { + if (!fs::exists(p) || !fs::is_directory(p)) throw; } @@ -727,11 +776,11 @@ void ShrinkDebugFile() // Amount of debug.log to save at end when shrinking (must fit in memory) constexpr size_t RECENT_DEBUG_HISTORY_SIZE = 10 * 1000000; // Scroll debug.log if it's getting too big - boost::filesystem::path pathLog = GetDataDir() / "debug.log"; - FILE* file = fopen(pathLog.string().c_str(), "r"); + fs::path pathLog = GetDataDir() / "debug.log"; + FILE* file = fsbridge::fopen(pathLog, "r"); // If debug.log file is more than 10% bigger the RECENT_DEBUG_HISTORY_SIZE // trim it down by saving only the last RECENT_DEBUG_HISTORY_SIZE bytes - if (file && boost::filesystem::file_size(pathLog) > 11 * (RECENT_DEBUG_HISTORY_SIZE / 10)) + if (file && fs::file_size(pathLog) > 11 * (RECENT_DEBUG_HISTORY_SIZE / 10)) { // Restart the file with some of the end std::vector<char> vch(RECENT_DEBUG_HISTORY_SIZE, 0); @@ -739,7 +788,7 @@ void ShrinkDebugFile() int nBytes = fread(vch.data(), 1, vch.size(), file); fclose(file); - file = fopen(pathLog.string().c_str(), "w"); + file = fsbridge::fopen(pathLog, "w"); if (file) { fwrite(vch.data(), 1, nBytes, file); @@ -751,10 +800,8 @@ void ShrinkDebugFile() } #ifdef WIN32 -boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate) +fs::path GetSpecialFolderPath(int nFolder, bool fCreate) { - namespace fs = boost::filesystem; - char pszPath[MAX_PATH] = ""; if(SHGetSpecialFolderPathA(NULL, pszPath, nFolder, fCreate)) @@ -792,6 +839,16 @@ void RenameThread(const char* name) void SetupEnvironment() { +#ifdef HAVE_MALLOPT_ARENA_MAX + // glibc-specific: On 32-bit systems set the number of arenas to 1. + // By default, since glibc 2.10, the C library will create up to two heap + // arenas per core. This is known to cause excessive virtual address space + // usage in our usage. Work around it by setting the maximum number of + // arenas to 1. + if (sizeof(void*) == 4) { + mallopt(M_ARENA_MAX, 1); + } +#endif // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale // may be invalid, in which case the "C" locale is used as fallback. #if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && !defined(__OpenBSD__) @@ -804,9 +861,9 @@ void SetupEnvironment() // The path locale is lazy initialized and to avoid deinitialization errors // in multithreading environments, it is set explicitly by the main thread. // A dummy locale is used to extract the internal default locale, used by - // boost::filesystem::path, which is then used to explicitly imbue the path. - std::locale loc = boost::filesystem::path::imbue(std::locale::classic()); - boost::filesystem::path::imbue(loc); + // fs::path, which is then used to explicitly imbue the path. + std::locale loc = fs::path::imbue(std::locale::classic()); + fs::path::imbue(loc); } bool SetupNetworking() diff --git a/src/util.h b/src/util.h index 61dc0d366c..ed28070a3f 100644 --- a/src/util.h +++ b/src/util.h @@ -15,6 +15,7 @@ #endif #include "compat.h" +#include "fs.h" #include "tinyformat.h" #include "utiltime.h" @@ -25,7 +26,6 @@ #include <string> #include <vector> -#include <boost/filesystem/path.hpp> #include <boost/signals2/signal.hpp> #include <boost/thread/exceptions.hpp> @@ -42,7 +42,6 @@ public: }; extern const std::map<std::string, std::vector<std::string> >& mapMultiArgs; -extern bool fDebug; extern bool fPrintToConsole; extern bool fPrintToDebugLog; @@ -55,6 +54,8 @@ extern CTranslationInterface translationInterface; extern const char * const BITCOIN_CONF_FILENAME; extern const char * const BITCOIN_PID_FILENAME; +extern std::atomic<uint32_t> logCategories; + /** * Translation function: Call Translate signal on UI interface, which returns a boost::optional result. * If no translation slot is registered, nothing is returned, and simply return the input. @@ -68,8 +69,54 @@ inline std::string _(const char* psz) void SetupEnvironment(); bool SetupNetworking(); +struct CLogCategoryActive +{ + std::string category; + bool active; +}; + +namespace BCLog { + enum LogFlags : uint32_t { + NONE = 0, + NET = (1 << 0), + TOR = (1 << 1), + MEMPOOL = (1 << 2), + HTTP = (1 << 3), + BENCH = (1 << 4), + ZMQ = (1 << 5), + DB = (1 << 6), + RPC = (1 << 7), + ESTIMATEFEE = (1 << 8), + ADDRMAN = (1 << 9), + SELECTCOINS = (1 << 10), + REINDEX = (1 << 11), + CMPCTBLOCK = (1 << 12), + RAND = (1 << 13), + PRUNE = (1 << 14), + PROXY = (1 << 15), + MEMPOOLREJ = (1 << 16), + LIBEVENT = (1 << 17), + COINDB = (1 << 18), + QT = (1 << 19), + LEVELDB = (1 << 20), + ALL = ~(uint32_t)0, + }; +} /** Return true if log accepts specified category */ -bool LogAcceptCategory(const char* category); +static inline bool LogAcceptCategory(uint32_t category) +{ + return (logCategories.load(std::memory_order_relaxed) & category) != 0; +} + +/** Returns a string with the log categories. */ +std::string ListLogCategories(); + +/** Returns a vector of the active log categories. */ +std::vector<CLogCategoryActive> ListActiveLogCategories(); + +/** Return true if str parses as a log category and set the flags in f */ +bool GetLogCategory(uint32_t *f, const std::string *str); + /** Send a string to the log output */ int LogPrintStr(const std::string &str); @@ -106,19 +153,19 @@ void FileCommit(FILE *file); bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length); -bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest); -bool TryCreateDirectory(const boost::filesystem::path& p); -boost::filesystem::path GetDefaultDataDir(); -const boost::filesystem::path &GetDataDir(bool fNetSpecific = true); +bool RenameOver(fs::path src, fs::path dest); +bool TryCreateDirectory(const fs::path& p); +fs::path GetDefaultDataDir(); +const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); -boost::filesystem::path GetConfigFile(const std::string& confPath); +fs::path GetConfigFile(const std::string& confPath); #ifndef WIN32 -boost::filesystem::path GetPidFile(); -void CreatePidFile(const boost::filesystem::path &path, pid_t pid); +fs::path GetPidFile(); +void CreatePidFile(const fs::path &path, pid_t pid); #endif void ReadConfigFile(const std::string& confPath); #ifdef WIN32 -boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate = true); +fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif void OpenDebugLog(); void ShrinkDebugFile(); diff --git a/src/validation.cpp b/src/validation.cpp index 037b9c0abc..6c60be45a1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -6,12 +6,14 @@ #include "validation.h" #include "arith_uint256.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" #include "consensus/consensus.h" #include "consensus/merkle.h" #include "consensus/validation.h" +#include "fs.h" #include "hash.h" #include "init.h" #include "policy/fees.h" @@ -41,8 +43,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/join.hpp> -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> #include <boost/math/distributions/poisson.hpp> #include <boost/thread.hpp> @@ -81,7 +81,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool; +CBlockPolicyEstimator feeEstimator; +CTxMemPool mempool(&feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -155,39 +156,6 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -/* Use this class to start tracking transactions that are removed from the - * mempool and pass all those transactions through SyncTransaction when the - * object goes out of scope. This is currently only used to call SyncTransaction - * on conflicts removed from the mempool during block connection. Applied in - * ActivateBestChain around ActivateBestStep which in turn calls: - * ConnectTip->removeForBlock->removeConflicts - */ -class MemPoolConflictRemovalTracker -{ -private: - std::vector<CTransactionRef> conflictedTxs; - CTxMemPool &pool; - -public: - MemPoolConflictRemovalTracker(CTxMemPool &_pool) : pool(_pool) { - pool.NotifyEntryRemoved.connect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - } - - void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { - if (reason == MemPoolRemovalReason::CONFLICT) { - conflictedTxs.push_back(txRemoved); - } - } - - ~MemPoolConflictRemovalTracker() { - pool.NotifyEntryRemoved.disconnect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - for (const auto& tx : conflictedTxs) { - GetMainSignals().SyncTransaction(*tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } - conflictedTxs.clear(); - } -}; - CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { // Find the first block the caller has in the main chain @@ -540,8 +508,9 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fChe void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { int expired = pool.Expire(GetTime() - age); - if (expired != 0) - LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); + if (expired != 0) { + LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); + } std::vector<uint256> vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); @@ -955,7 +924,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // Remove conflicting transactions from the mempool BOOST_FOREACH(const CTxMemPool::txiter it, allConflicting) { - LogPrint("mempool", "replacing tx %s with %s for %s BTC additional fees, %d delta bytes\n", + LogPrint(BCLog::MEMPOOL, "replacing tx %s with %s for %s BTC additional fees, %d delta bytes\n", it->GetTx().GetHash().ToString(), hash.ToString(), FormatMoney(nModifiedFees - nConflictingFees), @@ -982,7 +951,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - GetMainSignals().SyncTransaction(tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + GetMainSignals().TransactionAddedToMempool(ptx); return true; } @@ -1557,7 +1526,11 @@ bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint return fClean; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean + * will be true if no problems were found. Otherwise, the return value will be false in case + * of problems. Note that in any case, coins may be modified. */ +static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean = NULL) { assert(pindex->GetBlockHash() == view.GetBestBlock()); @@ -1710,8 +1683,11 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) { AssertLockHeld(cs_main); assert(pindex); @@ -1763,7 +1739,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; - LogPrint("bench", " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); + LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. @@ -1830,7 +1806,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; - LogPrint("bench", " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); + LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); CBlockUndo blockundo; @@ -1904,7 +1880,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; - LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); + LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) @@ -1914,9 +1890,9 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) - return state.DoS(100, false); + return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; - LogPrint("bench", " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); + LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); if (fJustCheck) return true; @@ -1948,16 +1924,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin view.SetBestBlock(pindex->GetBlockHash()); int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; - LogPrint("bench", " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); - - // Watch for changes to the previous coinbase transaction. - static uint256 hashPrevBestCoinBase; - GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0]->GetHash(); - + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; - LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); + LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); return true; } @@ -2005,10 +1975,11 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n nLastSetChain = nNow; } int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 100 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - 100 * 1024 * 1024); + // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), + std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -2152,7 +2123,8 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); // Read block from disk. - CBlock block; + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + CBlock& block = *pblock; if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); // Apply the block atomically to the chain state. @@ -2164,7 +2136,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara bool flushed = view.Flush(); assert(flushed); } - LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); + LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; @@ -2194,9 +2166,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - for (const auto& tx : block.vtx) { - GetMainSignals().SyncTransaction(*tx, pindexDelete->pprev, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } + GetMainSignals().BlockDisconnected(pblock); return true; } @@ -2206,40 +2176,96 @@ static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; +struct PerBlockConnectTrace { + CBlockIndex* pindex = NULL; + std::shared_ptr<const CBlock> pblock; + std::shared_ptr<std::vector<CTransactionRef>> conflictedTxs; + PerBlockConnectTrace() : conflictedTxs(std::make_shared<std::vector<CTransactionRef>>()) {} +}; /** * Used to track blocks whose transactions were applied to the UTXO state as a * part of a single ActivateBestChainStep call. + * + * This class also tracks transactions that are removed from the mempool as + * conflicts (per block) and can be used to pass all those transactions + * through SyncTransaction. + * + * This class assumes (and asserts) that the conflicted transactions for a given + * block are added via mempool callbacks prior to the BlockConnected() associated + * with those transactions. If any transactions are marked conflicted, it is + * assumed that an associated block will always be added. + * + * This class is single-use, once you call GetBlocksConnected() you have to throw + * it away and make a new one. */ -struct ConnectTrace { - std::vector<std::pair<CBlockIndex*, std::shared_ptr<const CBlock> > > blocksConnected; +class ConnectTrace { +private: + std::vector<PerBlockConnectTrace> blocksConnected; + CTxMemPool &pool; + +public: + ConnectTrace(CTxMemPool &_pool) : blocksConnected(1), pool(_pool) { + pool.NotifyEntryRemoved.connect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); + } + + ~ConnectTrace() { + pool.NotifyEntryRemoved.disconnect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); + } + + void BlockConnected(CBlockIndex* pindex, std::shared_ptr<const CBlock> pblock) { + assert(!blocksConnected.back().pindex); + assert(pindex); + assert(pblock); + blocksConnected.back().pindex = pindex; + blocksConnected.back().pblock = std::move(pblock); + blocksConnected.emplace_back(); + } + + std::vector<PerBlockConnectTrace>& GetBlocksConnected() { + // We always keep one extra block at the end of our list because + // blocks are added after all the conflicted transactions have + // been filled in. Thus, the last entry should always be an empty + // one waiting for the transactions from the next block. We pop + // the last entry here to make sure the list we return is sane. + assert(!blocksConnected.back().pindex); + assert(blocksConnected.back().conflictedTxs->empty()); + blocksConnected.pop_back(); + return blocksConnected; + } + + void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { + assert(!blocksConnected.back().pindex); + if (reason == MemPoolRemovalReason::CONFLICT) { + blocksConnected.back().conflictedTxs->emplace_back(std::move(txRemoved)); + } + } }; /** * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. * - * The block is always added to connectTrace (either after loading from disk or by copying - * pblock) - if that is not intended, care must be taken to remove the last entry in - * blocksConnected in case of failure. + * The block is added to connectTrace if connection succeeds. */ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); + std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - connectTrace.blocksConnected.emplace_back(pindexNew, pblockNew); if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); + pthisBlock = pblockNew; } else { - connectTrace.blocksConnected.emplace_back(pindexNew, pblock); + pthisBlock = pblock; } - const CBlock& blockConnecting = *connectTrace.blocksConnected.back().second; + const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; - LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pcoinsTip); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); @@ -2250,25 +2276,27 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, return error("ConnectTip(): ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; - LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); + LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; - LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); + LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; - LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); + LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; - LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); - LogPrint("bench", "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); + LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + + connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } @@ -2387,8 +2415,6 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c state = CValidationState(); fInvalidFound = true; fContinue = false; - // If we didn't actually connect the block, don't notify listeners about it - connectTrace.blocksConnected.pop_back(); break; } else { // A system error occurred (disk space, database error, ...). @@ -2460,18 +2486,11 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, break; const CBlockIndex *pindexFork; - ConnectTrace connectTrace; bool fInitialDownload; { LOCK(cs_main); - { // TODO: Temporarily ensure that mempool removals are notified before - // connected transactions. This shouldn't matter, but the abandoned - // state of transactions in our wallet is currently cleared when we - // receive another notification and there is a race condition where - // notification of a connected conflict might cause an outside process - // to abandon a transaction and then have it inadvertently cleared by - // the notification that the conflicted transaction was evicted. - MemPoolConflictRemovalTracker mrt(mempool); + ConnectTrace connectTrace(mempool); // Destructed before cs_main is unlocked + CBlockIndex *pindexOldTip = chainActive.Tip(); if (pindexMostWork == NULL) { pindexMostWork = FindMostWorkChain(); @@ -2494,16 +2513,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); - // throw all transactions though the signal-interface - - } // MemPoolConflictRemovalTracker destroyed and conflict evictions are notified - - // Transactions in the connected block are notified - for (const auto& pair : connectTrace.blocksConnected) { - assert(pair.second); - const CBlock& block = *(pair.second); - for (unsigned int i = 0; i < block.vtx.size(); i++) - GetMainSignals().SyncTransaction(*block.vtx[i], pair.first, i); + for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { + assert(trace.pblock && trace.pindex); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2667,7 +2679,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2675,7 +2687,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -2881,10 +2893,12 @@ static bool CheckIndexAgainstCheckpoint(const CBlockIndex* pindexPrev, CValidati return true; int nHeight = pindexPrev->nHeight+1; - // Don't accept any forks from the main chain prior to last checkpoint + // Don't accept any forks from the main chain prior to last checkpoint. + // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our + // MapBlockIndex. CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(chainparams.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight)); + return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); return true; } @@ -2900,9 +2914,11 @@ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& pa static int GetWitnessCommitmentIndex(const CBlock& block) { int commitpos = -1; - for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { - if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { - commitpos = o; + if (!block.vtx.empty()) { + for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { + if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { + commitpos = o; + } } } return commitpos; @@ -3083,7 +3099,7 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state CBlockIndex* pindexPrev = NULL; BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) - return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); + return state.DoS(10, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); @@ -3193,7 +3209,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (dbp == NULL) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) AbortNode(state, "Failed to write block"); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); @@ -3314,8 +3330,8 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { CDiskBlockPos pos(*it, 0); - boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); - boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + fs::remove(GetBlockPosFilename(pos, "blk")); + fs::remove(GetBlockPosFilename(pos, "rev")); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3391,7 +3407,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight } } - LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget/1024/1024, nCurrentUsage/1024/1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, nLastBlockWeCanPrune, count); @@ -3399,7 +3415,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight bool CheckDiskSpace(uint64_t nAdditionalBytes) { - uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; + uint64_t nFreeBytesAvailable = fs::space(GetDataDir()).available; // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) @@ -3412,11 +3428,11 @@ FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { if (pos.IsNull()) return NULL; - boost::filesystem::path path = GetBlockPosFilename(pos, prefix); - boost::filesystem::create_directories(path.parent_path()); - FILE* file = fopen(path.string().c_str(), "rb+"); + fs::path path = GetBlockPosFilename(pos, prefix); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, "rb+"); if (!file && !fReadOnly) - file = fopen(path.string().c_str(), "wb+"); + file = fsbridge::fopen(path, "wb+"); if (!file) { LogPrintf("Unable to open file %s\n", path.string()); return NULL; @@ -3439,7 +3455,7 @@ FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "rev", fReadOnly); } -boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) { return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); } @@ -3824,7 +3840,7 @@ bool InitBlockIndex(const CChainParams& chainparams) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) return error("LoadBlockIndex(): writing genesis block to disk failed"); CBlockIndex *pindex = AddToBlockIndex(block); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("LoadBlockIndex(): genesis block not accepted"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); @@ -3885,7 +3901,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB // detect out of order blocks, and store them for later uint256 hash = block.GetHash(); if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { - LogPrint("reindex", "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), + LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); @@ -3901,7 +3917,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB if (state.IsError()) break; } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { - LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); } // Activate the genesis block so normal node progress can continue @@ -3926,7 +3942,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { - LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), + LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); CValidationState dummy; @@ -4164,7 +4180,7 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(void) { int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; - FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "rb"); + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); @@ -4244,7 +4260,7 @@ void DumpMempool(void) int64_t mid = GetTimeMicros(); try { - FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "wb"); + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat.new", "wb"); if (!filestr) { return; } diff --git a/src/validation.h b/src/validation.h index 5f8e9a689c..c0f9b6d513 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,8 +11,8 @@ #endif #include "amount.h" -#include "chain.h" #include "coins.h" +#include "fs.h" #include "protocol.h" // For CMessageHeader::MessageStartChars #include "script/script_error.h" #include "sync.h" @@ -30,7 +30,6 @@ #include <atomic> #include <boost/unordered_map.hpp> -#include <boost/filesystem/path.hpp> class CBlockIndex; class CBlockTreeDB; @@ -39,6 +38,7 @@ class CChainParams; class CInv; class CConnman; class CScriptCheck; +class CBlockPolicyEstimator; class CTxMemPool; class CValidationInterface; class CValidationState; @@ -152,6 +152,7 @@ struct BlockHasher extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; +extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef boost::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap mapBlockIndex; @@ -250,7 +251,7 @@ FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Open an undo file (rev?????.dat) */ FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); /** Import blocks from an external file */ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = NULL); /** Initialize a new block tree database + block data on disk */ @@ -483,18 +484,6 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); -/** Apply the effects of this block (with given index) on the UTXO set represented by coins. - * Validity checks that depend on the UTXO set are also done; ConnectBlock() - * can fail if those validity checks fail (among other reasons). */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, - const CChainParams& chainparams, bool fJustCheck = false); - -/** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& coins, bool* pfClean = NULL); - /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index d4121a28bc..46d7c9b329 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -14,39 +14,39 @@ CMainSignals& GetMainSignals() void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); - g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); - g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); + g_signals.TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.ScriptForMining.connect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); - g_signals.BlockFound.connect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.NewPoWValidBlock.connect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.BlockFound.disconnect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.ScriptForMining.disconnect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); g_signals.BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.SyncTransaction.disconnect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); + g_signals.TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterAllValidationInterfaces() { - g_signals.BlockFound.disconnect_all_slots(); g_signals.ScriptForMining.disconnect_all_slots(); g_signals.BlockChecked.disconnect_all_slots(); g_signals.Broadcast.disconnect_all_slots(); g_signals.Inventory.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); - g_signals.SyncTransaction.disconnect_all_slots(); + g_signals.TransactionAddedToMempool.disconnect_all_slots(); + g_signals.BlockConnected.disconnect_all_slots(); + g_signals.BlockDisconnected.disconnect_all_slots(); g_signals.UpdatedBlockTip.disconnect_all_slots(); g_signals.NewPoWValidBlock.disconnect_all_slots(); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 7f13a29d23..460aecf243 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -7,16 +7,16 @@ #define BITCOIN_VALIDATIONINTERFACE_H #include <boost/signals2/signal.hpp> -#include <boost/shared_ptr.hpp> #include <memory> +#include "primitives/transaction.h" // CTransaction(Ref) + class CBlock; class CBlockIndex; struct CBlockLocator; class CBlockIndex; class CConnman; class CReserveScript; -class CTransaction; class CValidationInterface; class CValidationState; class uint256; @@ -33,14 +33,14 @@ void UnregisterAllValidationInterfaces(); class CValidationInterface { protected: virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - virtual void SyncTransaction(const CTransaction &tx, const CBlockIndex *pindex, int posInBlock) {} + virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} + virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} virtual void SetBestChain(const CBlockLocator &locator) {} - virtual void UpdatedTransaction(const uint256 &hash) {} virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} virtual void BlockChecked(const CBlock&, const CValidationState&) {} - virtual void GetScriptForMining(boost::shared_ptr<CReserveScript>&) {}; - virtual void ResetRequestCount(const uint256 &hash) {}; + virtual void GetScriptForMining(std::shared_ptr<CReserveScript>&) {}; virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); @@ -50,19 +50,15 @@ protected: struct CMainSignals { /** Notifies listeners of updated block chain tip */ boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; - /** A posInBlock value for SyncTransaction calls for transactions not - * included in connected blocks such as transactions removed from mempool, - * accepted to mempool or appearing in disconnected blocks.*/ - static const int SYNC_TRANSACTION_NOT_IN_BLOCK = -1; - /** Notifies listeners of updated transaction data (transaction, and - * optionally the block it is found in). Called with block data when - * transaction is included in a connected block, and without block data when - * transaction was accepted to mempool, removed from mempool (only when - * removal was due to conflict from connected block), or appeared in a - * disconnected block.*/ - boost::signals2::signal<void (const CTransaction &, const CBlockIndex *pindex, int posInBlock)> SyncTransaction; - /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ - boost::signals2::signal<void (const uint256 &)> UpdatedTransaction; + /** Notifies listeners of a transaction having been added to mempool. */ + boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; + /** + * Notifies listeners of a block being connected. + * Provides a vector of transactions evicted from the mempool as a result. + */ + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &)> BlockConnected; + /** Notifies listeners of a block being disconnected */ + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; /** Notifies listeners of a new active block chain. */ boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; /** Notifies listeners about an inventory item being seen on the network. */ @@ -77,9 +73,7 @@ struct CMainSignals { */ boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; /** Notifies listeners that a key for mining is required (coinbase) */ - boost::signals2::signal<void (boost::shared_ptr<CReserveScript>&)> ScriptForMining; - /** Notifies listeners that a block has been successfully mined */ - boost::signals2::signal<void (const uint256 &)> BlockFound; + boost::signals2::signal<void (std::shared_ptr<CReserveScript>&)> ScriptForMining; /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 0801bd7300..f47fc92b57 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -6,6 +6,7 @@ #include "db.h" #include "addrman.h" +#include "fs.h" #include "hash.h" #include "protocol.h" #include "util.h" @@ -17,7 +18,6 @@ #include <sys/stat.h> #endif -#include <boost/filesystem.hpp> #include <boost/foreach.hpp> #include <boost/thread.hpp> #include <boost/version.hpp> @@ -66,7 +66,7 @@ void CDBEnv::Close() EnvShutdown(); } -bool CDBEnv::Open(const boost::filesystem::path& pathIn) +bool CDBEnv::Open(const fs::path& pathIn) { if (fDbEnvInit) return true; @@ -74,9 +74,9 @@ bool CDBEnv::Open(const boost::filesystem::path& pathIn) boost::this_thread::interruption_point(); strPath = pathIn.string(); - boost::filesystem::path pathLogDir = pathIn / "database"; + fs::path pathLogDir = pathIn / "database"; TryCreateDirectory(pathLogDir); - boost::filesystem::path pathErrorFile = pathIn / "db.log"; + fs::path pathErrorFile = pathIn / "db.log"; LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); unsigned int nEnvFlags = 0; @@ -89,7 +89,7 @@ bool CDBEnv::Open(const boost::filesystem::path& pathIn) dbenv->set_lg_max(1048576); dbenv->set_lk_max_locks(40000); dbenv->set_lk_max_objects(40000); - dbenv->set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug + dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug dbenv->set_flags(DB_AUTO_COMMIT, 1); dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1); @@ -118,7 +118,7 @@ void CDBEnv::MakeMock() boost::this_thread::interruption_point(); - LogPrint("db", "CDBEnv::MakeMock\n"); + LogPrint(BCLog::DB, "CDBEnv::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); @@ -227,13 +227,13 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco return fSuccess; } -bool CDB::VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr) +bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) { LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); LogPrintf("Using wallet %s\n", walletFile); // Wallet file must be a plain filename without a directory - if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile)) + if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); return false; @@ -242,12 +242,12 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const boost::filesyst if (!bitdb.Open(dataDir)) { // try moving the database env out of the way - boost::filesystem::path pathDatabase = dataDir / "database"; - boost::filesystem::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + fs::path pathDatabase = dataDir / "database"; + fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); try { - boost::filesystem::rename(pathDatabase, pathDatabaseBak); + fs::rename(pathDatabase, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); - } catch (const boost::filesystem::filesystem_error&) { + } catch (const fs::filesystem_error&) { // failure is ok (well, not really, but it's not worse than what we started with) } @@ -261,9 +261,9 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const boost::filesyst return true; } -bool CDB::VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)) +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)) { - if (boost::filesystem::exists(dataDir / walletFile)) + if (fs::exists(dataDir / walletFile)) { CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc); if (r == CDBEnv::RECOVER_OK) @@ -560,7 +560,7 @@ void CDBEnv::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use - LogPrint("db", "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) return; { @@ -569,28 +569,28 @@ void CDBEnv::Flush(bool fShutdown) while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; - LogPrint("db", "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + LogPrint(BCLog::DB, "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); - LogPrint("db", "CDBEnv::Flush: %s checkpoint\n", strFile); + LogPrint(BCLog::DB, "CDBEnv::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); - LogPrint("db", "CDBEnv::Flush: %s detach\n", strFile); + LogPrint(BCLog::DB, "CDBEnv::Flush: %s detach\n", strFile); if (!fMockDb) dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint("db", "CDBEnv::Flush: %s closed\n", strFile); + LogPrint(BCLog::DB, "CDBEnv::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else mi++; } - LogPrint("db", "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) - boost::filesystem::remove_all(boost::filesystem::path(strPath) / "database"); + fs::remove_all(fs::path(strPath) / "database"); } } } @@ -617,7 +617,7 @@ bool CDB::PeriodicFlush(std::string strFile) std::map<std::string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile); if (mi != bitdb.mapFileUseCount.end()) { - LogPrint("db", "Flushing %s\n", strFile); + LogPrint(BCLog::DB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained @@ -625,7 +625,7 @@ bool CDB::PeriodicFlush(std::string strFile) bitdb.CheckpointLSN(strFile); bitdb.mapFileUseCount.erase(mi++); - LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } diff --git a/src/wallet/db.h b/src/wallet/db.h index 19c54e314c..9f912f9a1a 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_DB_H #include "clientversion.h" +#include "fs.h" #include "serialize.h" #include "streams.h" #include "sync.h" @@ -16,8 +17,6 @@ #include <string> #include <vector> -#include <boost/filesystem/path.hpp> - #include <db_cxx.h> static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; @@ -28,7 +27,7 @@ class CDBEnv private: bool fDbEnvInit; bool fMockDb; - // Don't change into boost::filesystem::path, as that can result in + // Don't change into fs::path, as that can result in // shutdown problems/crashes caused by a static initialized internal pointer. std::string strPath; @@ -67,7 +66,7 @@ public: typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; bool Salvage(const std::string& strFile, bool fAggressive, std::vector<KeyValPair>& vResult); - bool Open(const boost::filesystem::path& path); + bool Open(const fs::path& path); void Close(); void Flush(bool fShutdown); void CheckpointLSN(const std::string& strFile); @@ -110,9 +109,9 @@ public: ideal to be called periodically */ static bool PeriodicFlush(std::string strFile); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)); private: CDB(const CDB&); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp new file mode 100644 index 0000000000..82e5974065 --- /dev/null +++ b/src/wallet/feebumper.cpp @@ -0,0 +1,284 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "consensus/validation.h" +#include "wallet/feebumper.h" +#include "wallet/wallet.h" +#include "policy/fees.h" +#include "policy/policy.h" +#include "policy/rbf.h" +#include "validation.h" //for mempool access +#include "txmempool.h" +#include "utilmoneystr.h" +#include "util.h" +#include "net.h" + +// Calculate the size of the transaction assuming all signatures are max size +// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. +// TODO: re-use this in CWallet::CreateTransaction (right now +// CreateTransaction uses the constructed dummy-signed tx to do a priority +// calculation, but we should be able to refactor after priority is removed). +// NOTE: this requires that all inputs must be in mapWallet (eg the tx should +// be IsAllFromMe). +int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWallet) +{ + CMutableTransaction txNew(tx); + std::vector<CInputCoin> vCoins; + // Look up the inputs. We should have already checked that this transaction + // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our + // wallet, with a valid index into the vout array. + for (auto& input : tx.vin) { + const auto mi = pWallet->mapWallet.find(input.prevout.hash); + assert(mi != pWallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); + vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n)); + } + if (!pWallet->DummySignTx(txNew, vCoins)) { + // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) + // implies that we can sign for every input. + return -1; + } + return GetVirtualTransactionSize(txNew); +} + +CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable) + : + txid(std::move(txidIn)), + nOldFee(0), + nNewFee(0) +{ + vErrors.clear(); + bumpedTxid.SetNull(); + AssertLockHeld(pWallet->cs_wallet); + if (!pWallet->mapWallet.count(txid)) { + vErrors.push_back("Invalid or non-wallet transaction id"); + currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; + return; + } + auto it = pWallet->mapWallet.find(txid); + const CWalletTx& wtx = it->second; + + if (pWallet->HasWalletSpend(txid)) { + vErrors.push_back("Transaction has descendants in the wallet"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + + { + LOCK(mempool.cs); + auto it_mp = mempool.mapTx.find(txid); + if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + vErrors.push_back("Transaction has descendants in the mempool"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + } + + if (wtx.GetDepthInMainChain() != 0) { + vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + if (!SignalsOptInRBF(wtx)) { + vErrors.push_back("Transaction is not BIP 125 replaceable"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + if (wtx.mapValue.count("replaced_by_txid")) { + vErrors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // check that original tx consists entirely of our inputs + // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) + if (!pWallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { + vErrors.push_back("Transaction contains inputs that don't belong to this wallet"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // figure out which output was change + // if there was no change output or multiple change outputs, fail + int nOutput = -1; + for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { + if (pWallet->IsChange(wtx.tx->vout[i])) { + if (nOutput != -1) { + vErrors.push_back("Transaction has multiple change outputs"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + nOutput = i; + } + } + if (nOutput == -1) { + vErrors.push_back("Transaction does not have a change output"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // Calculate the expected size of the new transaction. + int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); + const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, pWallet); + if (maxNewTxSize < 0) { + vErrors.push_back("Transaction contains inputs that cannot be signed"); + currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; + return; + } + + // calculate the old fee and fee-rate + nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + CFeeRate nOldFeeRate(nOldFee, txSize); + CFeeRate nNewFeeRate; + // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to + // future proof against changes to network wide policy for incremental relay + // fee that our node may not be aware of. + CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); + if (::incrementalRelayFee > walletIncrementalRelayFee) { + walletIncrementalRelayFee = ::incrementalRelayFee; + } + + if (totalFee > 0) { + CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); + if (totalFee < minTotalFee) { + vErrors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", + FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + CAmount requiredFee = CWallet::GetRequiredFee(maxNewTxSize); + if (totalFee < requiredFee) { + vErrors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", + FormatMoney(requiredFee))); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + nNewFee = totalFee; + nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); + } else { + // if user specified a confirm target then don't consider any global payTxFee + if (specifiedConfirmTarget) { + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, CAmount(0)); + } + // otherwise use the regular wallet logic to select payTxFee or default confirm target + else { + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator); + } + + nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); + + // New fee rate must be at least old rate + minimum incremental relay rate + // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized + // in that unit (fee per kb). + // However, nOldFeeRate is a calculated value from the tx fee/size, so + // add 1 satoshi to the result, because it may have been rounded down. + if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { + nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); + nNewFee = nNewFeeRate.GetFee(maxNewTxSize); + } + } + + // Check that in all cases the new fee doesn't violate maxTxFee + if (nNewFee > maxTxFee) { + vErrors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", + FormatMoney(nNewFee), FormatMoney(maxTxFee))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // check that fee rate is higher than mempool's minimum fee + // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) + // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, + // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a + // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. + CFeeRate minMempoolFeeRate = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { + vErrors.push_back(strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // Now modify the output to increase the fee. + // If the output is not large enough to pay the fee, fail. + CAmount nDelta = nNewFee - nOldFee; + assert(nDelta > 0); + mtx = *wtx.tx; + CTxOut* poutput = &(mtx.vout[nOutput]); + if (poutput->nValue < nDelta) { + vErrors.push_back("Change output is too small to bump the fee"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // If the output would become dust, discard it (converting the dust to fee) + poutput->nValue -= nDelta; + if (poutput->nValue <= poutput->GetDustThreshold(::dustRelayFee)) { + LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); + nNewFee += poutput->nValue; + mtx.vout.erase(mtx.vout.begin() + nOutput); + } + + // Mark new tx not replaceable, if requested. + if (!newTxReplaceable) { + for (auto& input : mtx.vin) { + if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; + } + } + + currentResult = BumpFeeResult::OK; +} + +bool CFeeBumper::signTransaction(CWallet *pWallet) +{ + return pWallet->SignTransaction(mtx); +} + +bool CFeeBumper::commit(CWallet *pWallet) +{ + AssertLockHeld(pWallet->cs_wallet); + if (!vErrors.empty() || currentResult != BumpFeeResult::OK) { + return false; + } + if (txid.IsNull() || !pWallet->mapWallet.count(txid)) { + vErrors.push_back("Invalid or non-wallet transaction id"); + currentResult = BumpFeeResult::MISC_ERROR; + return false; + } + CWalletTx& oldWtx = pWallet->mapWallet[txid]; + + CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); + // commit/broadcast the tx + CReserveKey reservekey(pWallet); + wtxBumped.mapValue = oldWtx.mapValue; + wtxBumped.mapValue["replaces_txid"] = oldWtx.GetHash().ToString(); + wtxBumped.vOrderForm = oldWtx.vOrderForm; + wtxBumped.strFromAccount = oldWtx.strFromAccount; + wtxBumped.fTimeReceivedIsTxTime = true; + wtxBumped.fFromMe = true; + CValidationState state; + if (!pWallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { + // NOTE: CommitTransaction never returns false, so this should never happen. + vErrors.push_back(strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); + return false; + } + + bumpedTxid = wtxBumped.GetHash(); + if (state.IsInvalid()) { + // This can happen if the mempool rejected the transaction. Report + // what happened in the "errors" response. + vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + } + + // mark the original tx as bumped + if (!pWallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { + // TODO: see if JSON-RPC has a standard way of returning a response + // along with an exception. It would be good to return information about + // wtxBumped to the caller even if marking the original transaction + // replaced does not succeed for some reason. + vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + } + return true; +} + diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h new file mode 100644 index 0000000000..681f55e4e5 --- /dev/null +++ b/src/wallet/feebumper.h @@ -0,0 +1,56 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_FEEBUMPER_H +#define BITCOIN_WALLET_FEEBUMPER_H + +#include <primitives/transaction.h> + +class CWallet; +class uint256; + +enum class BumpFeeResult +{ + OK, + INVALID_ADDRESS_OR_KEY, + INVALID_REQUEST, + INVALID_PARAMETER, + WALLET_ERROR, + MISC_ERROR, +}; + +class CFeeBumper +{ +public: + CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable); + BumpFeeResult getResult() const { return currentResult; } + const std::vector<std::string>& getErrors() const { return vErrors; } + CAmount getOldFee() const { return nOldFee; } + CAmount getNewFee() const { return nNewFee; } + uint256 getBumpedTxId() const { return bumpedTxid; } + + /* signs the new transaction, + * returns false if the tx couldn't be found or if it was + * impossible to create the signature(s) + */ + bool signTransaction(CWallet *pWallet); + + /* commits the fee bump, + * returns true, in case of CWallet::CommitTransaction was successful + * but, eventually sets vErrors if the tx could not be added to the mempool (will try later) + * or if the old transaction could not be marked as replaced + */ + bool commit(CWallet *pWalletNonConst); + +private: + const uint256 txid; + uint256 bumpedTxid; + CMutableTransaction mtx; + std::vector<std::string> vErrors; + BumpFeeResult currentResult; + CAmount nOldFee; + CAmount nNewFee; +}; + +#endif // BITCOIN_WALLET_FEEBUMPER_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 7ff9e7ae58..82708dab26 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -95,6 +95,8 @@ UniValue importprivkey(const JSONRPCRequest& request) + HelpExampleCli("importprivkey", "\"mykey\"") + "\nImport using a label and without rescan\n" + HelpExampleCli("importprivkey", "\"mykey\" \"testing\" false") + + "\nImport using default blank label and without rescan\n" + + HelpExampleCli("importprivkey", "\"mykey\" \"\" false") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") ); @@ -154,6 +156,31 @@ UniValue importprivkey(const JSONRPCRequest& request) return NullUniValue; } +UniValue abortrescan(const JSONRPCRequest& request) +{ + CWallet* const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 0) + throw std::runtime_error( + "abortrescan\n" + "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nExamples:\n" + "\nImport a private key\n" + + HelpExampleCli("importprivkey", "\"mykey\"") + + "\nAbort the running wallet rescan\n" + + HelpExampleCli("abortrescan", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("abortrescan", "") + ); + + if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; + pwallet->AbortRescan(); + return true; +} + void ImportAddress(CWallet*, const CBitcoinAddress& address, const std::string& strLabel); void ImportScript(CWallet* const pwallet, const CScript& script, const std::string& strLabel, bool isRedeemScript) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 84e7eb60d7..5cbeeab4eb 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -9,8 +9,10 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" +#include "wallet/coincontrol.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "rpc/server.h" @@ -18,8 +20,9 @@ #include "timedata.h" #include "util.h" #include "utilmoneystr.h" -#include "wallet.h" -#include "walletdb.h" +#include "wallet/feebumper.h" +#include "wallet/wallet.h" +#include "wallet/walletdb.h" #include <stdint.h> @@ -221,7 +224,7 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) CReserveKey reservekey(pwallet); CPubKey vchPubKey; - if (!reservekey.GetReservedKey(vchPubKey)) + if (!reservekey.GetReservedKey(vchPubKey, true)) throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); reservekey.KeepKey(); @@ -1852,7 +1855,7 @@ UniValue gettransaction(const JSONRPCRequest& request) " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" " 'send' category of transactions.\n" " \"abandoned\": xxx (bool) 'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" - " 'send' category of transactions.\n" + " 'send' category of transactions.\n" " }\n" " ,...\n" " ],\n" @@ -2418,16 +2421,17 @@ UniValue getwalletinfo(const JSONRPCRequest& request) "Returns an object containing various wallet state info.\n" "\nResult:\n" "{\n" - " \"walletversion\": xxxxx, (numeric) the wallet version\n" - " \"balance\": xxxxxxx, (numeric) the total confirmed balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" - " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" - " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" - " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" - " \"hdmasterkeyid\": \"<hash160>\" (string) the Hash160 of the HD master pubkey\n" + " \"walletversion\": xxxxx, (numeric) the wallet version\n" + " \"balance\": xxxxxxx, (numeric) the total confirmed balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated (only counts external keys)\n" + " \"keypoolsize_hd_internal\": xxxx, (numeric) how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)\n" + " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" + " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" + " \"hdmasterkeyid\": \"<hash160>\" (string) the Hash160 of the HD master pubkey\n" "}\n" "\nExamples:\n" + HelpExampleCli("getwalletinfo", "") @@ -2437,18 +2441,23 @@ UniValue getwalletinfo(const JSONRPCRequest& request) LOCK2(cs_main, pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); + + size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); obj.push_back(Pair("walletversion", pwallet->GetVersion())); obj.push_back(Pair("balance", ValueFromAmount(pwallet->GetBalance()))); obj.push_back(Pair("unconfirmed_balance", ValueFromAmount(pwallet->GetUnconfirmedBalance()))); obj.push_back(Pair("immature_balance", ValueFromAmount(pwallet->GetImmatureBalance()))); obj.push_back(Pair("txcount", (int)pwallet->mapWallet.size())); obj.push_back(Pair("keypoololdest", pwallet->GetOldestKeyPoolTime())); - obj.push_back(Pair("keypoolsize", (int)pwallet->GetKeyPoolSize())); + obj.push_back(Pair("keypoolsize", (int64_t)kpExternalSize)); + CKeyID masterKeyID = pwallet->GetHDChain().masterKeyID; + if (!masterKeyID.IsNull() && pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { + obj.push_back(Pair("keypoolsize_hd_internal", (int64_t)(pwallet->GetKeyPoolSize() - kpExternalSize))); + } if (pwallet->IsCrypted()) { obj.push_back(Pair("unlocked_until", pwallet->nRelockTime)); } obj.push_back(Pair("paytxfee", ValueFromAmount(payTxFee.GetFeePerK()))); - CKeyID masterKeyID = pwallet->GetHDChain().masterKeyID; if (!masterKeyID.IsNull()) obj.push_back(Pair("hdmasterkeyid", masterKeyID.GetHex())); return obj; @@ -2772,33 +2781,6 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) return result; } -// Calculate the size of the transaction assuming all signatures are max size -// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. -// TODO: re-use this in CWallet::CreateTransaction (right now -// CreateTransaction uses the constructed dummy-signed tx to do a priority -// calculation, but we should be able to refactor after priority is removed). -// NOTE: this requires that all inputs must be in mapWallet (eg the tx should -// be IsAllFromMe). -int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, CWallet &wallet) -{ - CMutableTransaction txNew(tx); - std::vector<std::pair<CWalletTx*, unsigned int>> vCoins; - // Look up the inputs. We should have already checked that this transaction - // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our - // wallet, with a valid index into the vout array. - for (auto& input : tx.vin) { - const auto mi = wallet.mapWallet.find(input.prevout.hash); - assert(mi != wallet.mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - vCoins.emplace_back(std::make_pair(&(mi->second), input.prevout.n)); - } - if (!wallet.DummySignTx(txNew, vCoins)) { - // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) - // implies that we can sign for every input. - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction contains inputs that cannot be signed"); - } - return GetVirtualTransactionSize(txNew); -} - UniValue bumpfee(const JSONRPCRequest& request) { CWallet * const pwallet = GetWalletForJSONRPCRequest(request); @@ -2853,63 +2835,6 @@ UniValue bumpfee(const JSONRPCRequest& request) uint256 hash; hash.SetHex(request.params[0].get_str()); - // retrieve the original tx from the wallet - LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(pwallet); - if (!pwallet->mapWallet.count(hash)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); - } - CWalletTx& wtx = pwallet->mapWallet[hash]; - - if (pwallet->HasWalletSpend(hash)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Transaction has descendants in the wallet"); - } - - { - LOCK(mempool.cs); - auto it = mempool.mapTx.find(hash); - if (it != mempool.mapTx.end() && it->GetCountWithDescendants() > 1) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Transaction has descendants in the mempool"); - } - } - - if (wtx.GetDepthInMainChain() != 0) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction has been mined, or is conflicted with a mined transaction"); - } - - if (!SignalsOptInRBF(wtx)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction is not BIP 125 replaceable"); - } - - if (wtx.mapValue.count("replaced_by_txid")) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Cannot bump transaction %s which was already bumped by transaction %s", hash.ToString(), wtx.mapValue.at("replaced_by_txid"))); - } - - // check that original tx consists entirely of our inputs - // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!pwallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction contains inputs that don't belong to this wallet"); - } - - // figure out which output was change - // if there was no change output or multiple change outputs, fail - int nOutput = -1; - for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { - if (pwallet->IsChange(wtx.tx->vout[i])) { - if (nOutput != -1) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction has multiple change outputs"); - } - nOutput = i; - } - } - if (nOutput == -1) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction does not have a change output"); - } - - // Calculate the expected size of the new transaction. - int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); - const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, *pwallet); - // optional parameters bool specifiedConfirmTarget = false; int newConfirmTarget = nTxConfirmTarget; @@ -2935,11 +2860,8 @@ UniValue bumpfee(const JSONRPCRequest& request) } } else if (options.exists("totalFee")) { totalFee = options["totalFee"].get_int64(); - CAmount requiredFee = CWallet::GetRequiredFee(maxNewTxSize); - if (totalFee < requiredFee ) { - throw JSONRPCError(RPC_INVALID_PARAMETER, - strprintf("Insufficient totalFee (cannot be less than required fee %s)", - FormatMoney(requiredFee))); + if (totalFee <= 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid totalFee %s (must be greater than 0)", FormatMoney(totalFee))); } } @@ -2948,148 +2870,53 @@ UniValue bumpfee(const JSONRPCRequest& request) } } - // calculate the old fee and fee-rate - CAmount nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); - CFeeRate nOldFeeRate(nOldFee, txSize); - CAmount nNewFee; - CFeeRate nNewFeeRate; - // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to - // future proof against changes to network wide policy for incremental relay - // fee that our node may not be aware of. - CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); - if (::incrementalRelayFee > walletIncrementalRelayFee) { - walletIncrementalRelayFee = ::incrementalRelayFee; - } - - if (totalFee > 0) { - CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); - if (totalFee < minTotalFee) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", - FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); - } - nNewFee = totalFee; - nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); - } else { - // if user specified a confirm target then don't consider any global payTxFee - if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, CAmount(0)); - } - // otherwise use the regular wallet logic to select payTxFee or default confirm target - else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool); - } - - nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); - - // New fee rate must be at least old rate + minimum incremental relay rate - // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized - // in that unit (fee per kb). - // However, nOldFeeRate is a calculated value from the tx fee/size, so - // add 1 satoshi to the result, because it may have been rounded down. - if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { - nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); - nNewFee = nNewFeeRate.GetFee(maxNewTxSize); - } - } + LOCK2(cs_main, pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); - // Check that in all cases the new fee doesn't violate maxTxFee - if (nNewFee > maxTxFee) { - throw JSONRPCError(RPC_WALLET_ERROR, - strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(nNewFee), FormatMoney(maxTxFee))); - } - - // check that fee rate is higher than mempool's minimum fee - // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) - // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, - // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a - // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. - CFeeRate minMempoolFeeRate = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); - if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); - } - - // Now modify the output to increase the fee. - // If the output is not large enough to pay the fee, fail. - CAmount nDelta = nNewFee - nOldFee; - assert(nDelta > 0); - CMutableTransaction tx(*(wtx.tx)); - CTxOut* poutput = &(tx.vout[nOutput]); - if (poutput->nValue < nDelta) { - throw JSONRPCError(RPC_WALLET_ERROR, "Change output is too small to bump the fee"); - } - - // If the output would become dust, discard it (converting the dust to fee) - poutput->nValue -= nDelta; - if (poutput->nValue <= poutput->GetDustThreshold(::dustRelayFee)) { - LogPrint("rpc", "Bumping fee and discarding dust output\n"); - nNewFee += poutput->nValue; - tx.vout.erase(tx.vout.begin() + nOutput); - } - - // Mark new tx not replaceable, if requested. - if (!replaceable) { - for (auto& input : tx.vin) { - if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; + CFeeBumper feeBump(pwallet, hash, newConfirmTarget, specifiedConfirmTarget, totalFee, replaceable); + BumpFeeResult res = feeBump.getResult(); + if (res != BumpFeeResult::OK) + { + switch(res) { + case BumpFeeResult::INVALID_ADDRESS_OR_KEY: + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, feeBump.getErrors()[0]); + break; + case BumpFeeResult::INVALID_REQUEST: + throw JSONRPCError(RPC_INVALID_REQUEST, feeBump.getErrors()[0]); + break; + case BumpFeeResult::INVALID_PARAMETER: + throw JSONRPCError(RPC_INVALID_PARAMETER, feeBump.getErrors()[0]); + break; + case BumpFeeResult::WALLET_ERROR: + throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + break; + default: + throw JSONRPCError(RPC_MISC_ERROR, feeBump.getErrors()[0]); + break; } } - // sign the new tx - CTransaction txNewConst(tx); - int nIn = 0; - for (auto& input : tx.vin) { - std::map<uint256, CWalletTx>::const_iterator mi = pwallet->mapWallet.find(input.prevout.hash); - assert(mi != pwallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; - const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; - SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(pwallet, &txNewConst, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); - } - UpdateTransaction(tx, nIn, sigdata); - nIn++; + // sign bumped transaction + if (!feeBump.signTransaction(pwallet)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } - - // commit/broadcast the tx - CReserveKey reservekey(pwallet); - CWalletTx wtxBumped(pwallet, MakeTransactionRef(std::move(tx))); - wtxBumped.mapValue = wtx.mapValue; - wtxBumped.mapValue["replaces_txid"] = hash.ToString(); - wtxBumped.vOrderForm = wtx.vOrderForm; - wtxBumped.strFromAccount = wtx.strFromAccount; - wtxBumped.fTimeReceivedIsTxTime = true; - wtxBumped.fFromMe = true; - CValidationState state; - if (!pwallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { - // NOTE: CommitTransaction never returns false, so this should never happen. - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); - } - - UniValue vErrors(UniValue::VARR); - if (state.IsInvalid()) { - // This can happen if the mempool rejected the transaction. Report - // what happened in the "errors" response. - vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + // commit the bumped transaction + if(!feeBump.commit(pwallet)) { + throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); } - - // mark the original tx as bumped - if (!pwallet->MarkReplaced(wtx.GetHash(), wtxBumped.GetHash())) { - // TODO: see if JSON-RPC has a standard way of returning a response - // along with an exception. It would be good to return information about - // wtxBumped to the caller even if marking the original transaction - // replaced does not succeed for some reason. - vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); - } - UniValue result(UniValue::VOBJ); - result.push_back(Pair("txid", wtxBumped.GetHash().GetHex())); - result.push_back(Pair("origfee", ValueFromAmount(nOldFee))); - result.push_back(Pair("fee", ValueFromAmount(nNewFee))); - result.push_back(Pair("errors", vErrors)); + result.push_back(Pair("txid", feeBump.getBumpedTxId().GetHex())); + result.push_back(Pair("origfee", ValueFromAmount(feeBump.getOldFee()))); + result.push_back(Pair("fee", ValueFromAmount(feeBump.getNewFee()))); + UniValue errors(UniValue::VARR); + for (const std::string& err: feeBump.getErrors()) + errors.push_back(err); + result.push_back(errors); return result; } +extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); extern UniValue importaddress(const JSONRPCRequest& request); @@ -3106,6 +2933,7 @@ static const CRPCCommand commands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false, {"hexstring","options"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, true, {} }, { "wallet", "abandontransaction", &abandontransaction, false, {"txid"} }, + { "wallet", "abortrescan", &abortrescan, false, {} }, { "wallet", "addmultisigaddress", &addmultisigaddress, true, {"nrequired","keys","account"} }, { "wallet", "addwitnessaddress", &addwitnessaddress, true, {"address"} }, { "wallet", "backupwallet", &backupwallet, true, {"destination"} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 67e5e90224..8eeba72a06 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -29,7 +29,7 @@ extern UniValue importmulti(const JSONRPCRequest& request); std::vector<std::unique_ptr<CWalletTx>> wtxn; -typedef std::set<std::pair<const CWalletTx*,unsigned int> > CoinSet; +typedef std::set<CInputCoin> CoinSet; BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) @@ -424,6 +424,17 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); ::pwalletMain = backup; } + + // Verify ScanForWalletTransactions does not return null when the scan is + // elided due to the nTimeFirstKey optimization. + { + CWallet wallet; + { + LOCK(wallet.cs_wallet); + wallet.UpdateTimeFirstKey(newTip->GetBlockTime() + 7200 + 1); + } + BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(newTip)); + } } // Check that GetImmatureCredit() returns a newly calculated value instead of diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 2f9d47aa0f..a8f07aba0d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -11,11 +11,14 @@ #include "wallet/coincontrol.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "fs.h" #include "key.h" #include "keystore.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" +#include "policy/rbf.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "script/script.h" @@ -30,7 +33,6 @@ #include <assert.h> #include <boost/algorithm/string/replace.hpp> -#include <boost/filesystem.hpp> #include <boost/thread.hpp> CWallet* pwalletMain = NULL; @@ -64,10 +66,10 @@ const uint256 CMerkleTx::ABANDON_HASH(uint256S("00000000000000000000000000000000 struct CompareValueOnly { - bool operator()(const std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> >& t1, - const std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> >& t2) const + bool operator()(const CInputCoin& t1, + const CInputCoin& t2) const { - return t1.first < t2.first; + return t1.txout.nValue < t2.txout.nValue; } }; @@ -85,7 +87,7 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const return &(it->second); } -CPubKey CWallet::GenerateNewKey() +CPubKey CWallet::GenerateNewKey(bool internal) { AssertLockHeld(cs_wallet); // mapKeyMetadata bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets @@ -98,7 +100,7 @@ CPubKey CWallet::GenerateNewKey() // use HD key derivation if HD was enabled during wallet creation if (IsHDEnabled()) { - DeriveNewChildKey(metadata, secret); + DeriveNewChildKey(metadata, secret, (CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); } else { secret.MakeNewKey(fCompressed); } @@ -118,13 +120,13 @@ CPubKey CWallet::GenerateNewKey() return pubkey; } -void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret) +void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal) { // for now we use a fixed keypath scheme of m/0'/0'/k CKey key; //master key seed (256bit) CExtKey masterKey; //hd master key CExtKey accountKey; //key at m/0' - CExtKey externalChainChildKey; //key at m/0'/0' + CExtKey chainChildKey; //key at m/0'/0' (external) or m/0'/1' (internal) CExtKey childKey; //key at m/0'/0'/<n>' // try to get the master key @@ -137,22 +139,28 @@ void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret) // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); - // derive m/0'/0' - accountKey.Derive(externalChainChildKey, BIP32_HARDENED_KEY_LIMIT); + // derive m/0'/0' (external chain) OR m/0'/1' (internal chain) + assert(internal ? CanSupportFeature(FEATURE_HD_SPLIT) : true); + accountKey.Derive(chainChildKey, BIP32_HARDENED_KEY_LIMIT+(internal ? 1 : 0)); // derive child key at next index, skip keys already known to the wallet do { // always derive hardened keys // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 - externalChainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; - metadata.hdMasterKeyID = hdChain.masterKeyID; - // increment childkey index - hdChain.nExternalChainCounter++; + if (internal) { + chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; + hdChain.nInternalChainCounter++; + } + else { + chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + hdChain.nExternalChainCounter++; + } } while (HaveKey(childKey.key.GetPubKey().GetID())); secret = childKey.key; - + metadata.hdMasterKeyID = hdChain.masterKeyID; // update the chain model in the database if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); @@ -633,7 +641,9 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) if (IsHDEnabled()) { CKey key; CPubKey masterPubKey = GenerateNewHDMasterKey(); - if (!SetHDMasterKey(masterPubKey)) + // preserve the old chains version to not break backward compatibility + CHDChain oldChain = GetHDChain(); + if (!SetHDMasterKey(masterPubKey, &oldChain)) return false; } @@ -799,7 +809,7 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo // Generate a new key if (bForceNew) { - if (!GetKeyFromPool(account.vchPubKey)) + if (!GetKeyFromPool(account.vchPubKey, false)) return false; SetAddressBook(account.vchPubKey.GetID(), strAccount, "receive"); @@ -948,9 +958,9 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) /** * Add a transaction to the wallet, or update it. pIndex and posInBlock should * be set when the transaction was known to be included in a block. When - * posInBlock = SYNC_TRANSACTION_NOT_IN_BLOCK (-1) , then wallet state is not - * updated in AddToWallet, but notifications happen and cached balances are - * marked dirty. + * pIndex == NULL, then wallet state is not updated in AddToWallet, but + * notifications happen and cached balances are marked dirty. + * * If fUpdate is true, existing transactions will be updated. * TODO: One exception to this is that the abandoned state is cleared under the * assumption that any further notification of a transaction that was considered @@ -958,12 +968,13 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. */ -bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate) { + const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (posInBlock != -1) { + if (pIndex != NULL) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { @@ -980,10 +991,10 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex if (fExisted && !fUpdate) return false; if (fExisted || IsMine(tx) || IsFromMe(tx)) { - CWalletTx wtx(this, MakeTransactionRef(tx)); + CWalletTx wtx(this, ptx); // Get merkle branch if transaction was found in a block - if (posInBlock != -1) + if (pIndex != NULL) wtx.SetMerkleBranch(pIndex, posInBlock); return AddToWallet(wtx, false); @@ -1108,11 +1119,10 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock) -{ - LOCK2(cs_main, cs_wallet); +void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pindex, int posInBlock) { + const CTransaction& tx = *ptx; - if (!AddToWalletIfInvolvingMe(tx, pindex, posInBlock, true)) + if (!AddToWalletIfInvolvingMe(ptx, pindex, posInBlock, true)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1125,6 +1135,38 @@ void CWallet::SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, } } +void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { + LOCK2(cs_main, cs_wallet); + SyncTransaction(ptx); +} + +void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { + LOCK2(cs_main, cs_wallet); + // TODO: Tempoarily ensure that mempool removals are notified before + // connected transactions. This shouldn't matter, but the abandoned + // state of transactions in our wallet is currently cleared when we + // receive another notification and there is a race condition where + // notification of a connected conflict might cause an outside process + // to abandon a transaction and then have it inadvertantly cleared by + // the notification that the conflicted transaction was evicted. + + for (const CTransactionRef& ptx : vtxConflicted) { + SyncTransaction(ptx); + } + for (size_t i = 0; i < pblock->vtx.size(); i++) { + SyncTransaction(pblock->vtx[i], pindex, i); + } +} + +void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { + LOCK2(cs_main, cs_wallet); + + for (const CTransactionRef& ptx : pblock->vtx) { + SyncTransaction(ptx); + } +} + + isminetype CWallet::IsMine(const CTxIn &txin) const { @@ -1300,17 +1342,17 @@ CPubKey CWallet::GenerateNewHDMasterKey() return pubkey; } -bool CWallet::SetHDMasterKey(const CPubKey& pubkey) +bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) { LOCK(cs_wallet); - - // ensure this wallet.dat can only be opened by clients supporting HD - SetMinVersion(FEATURE_HD); - // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object CHDChain newHdChain; + if (possibleOldChain) { + // preserve the old chains version + newHdChain.nVersion = possibleOldChain->nVersion; + } newHdChain.masterKeyID = pubkey.GetID(); SetHDChain(newHdChain, false); @@ -1474,18 +1516,21 @@ void CWalletTx::GetAccountAmounts(const std::string& strAccount, CAmount& nRecei * exist in the wallet will be updated. * * Returns pointer to the first block in the last contiguous range that was - * successfully scanned. - * + * successfully scanned or elided (elided if pIndexStart points at a block + * before CWallet::nTimeFirstKey). Returns null if there is no such range, or + * the range doesn't include chainActive.Tip(). */ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) { - CBlockIndex* ret = nullptr; int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); CBlockIndex* pindex = pindexStart; + CBlockIndex* ret = pindexStart; { LOCK2(cs_main, cs_wallet); + fAbortRescan = false; + fScanningWallet = true; // no need to read and scan block, if block was created before // our wallet birthday (as adjusted for block time variability) @@ -1495,7 +1540,7 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); - while (pindex) + while (pindex && !fAbortRescan) { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); @@ -1503,7 +1548,7 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - AddToWalletIfInvolvingMe(*block.vtx[posInBlock], pindex, posInBlock, fUpdate); + AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate); } if (!ret) { ret = pindex; @@ -1517,7 +1562,12 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } } + if (pindex && fAbortRescan) { + LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI + + fScanningWallet = false; } return ret; } @@ -1752,10 +1802,7 @@ CAmount CWalletTx::GetChange() const bool CWalletTx::InMempool() const { LOCK(mempool.cs); - if (mempool.exists(GetHash())) { - return true; - } - return false; + return mempool.exists(GetHash()); } bool CWalletTx::IsTrusted() const @@ -2023,7 +2070,7 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const } } -static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > >& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, +static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { std::vector<char> vfIncluded; @@ -2050,7 +2097,7 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair //the selection random. if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i]) { - nTotal += vValue[i].first; + nTotal += vValue[i].txout.nValue; vfIncluded[i] = true; if (nTotal >= nTargetValue) { @@ -2060,7 +2107,7 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair nBest = nTotal; vfBest = vfIncluded; } - nTotal -= vValue[i].first; + nTotal -= vValue[i].txout.nValue; vfIncluded[i] = false; } } @@ -2070,16 +2117,14 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair } bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMine, const int nConfTheirs, const uint64_t nMaxAncestors, std::vector<COutput> vCoins, - std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const + std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const { setCoinsRet.clear(); nValueRet = 0; // List of values less than target - std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > coinLowestLarger; - coinLowestLarger.first = std::numeric_limits<CAmount>::max(); - coinLowestLarger.second.first = NULL; - std::vector<std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > > vValue; + boost::optional<CInputCoin> coinLowestLarger; + std::vector<CInputCoin> vValue; CAmount nTotalLower = 0; random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); @@ -2098,22 +2143,21 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin continue; int i = output.i; - CAmount n = pcoin->tx->vout[i].nValue; - std::pair<CAmount,std::pair<const CWalletTx*,unsigned int> > coin = std::make_pair(n,std::make_pair(pcoin, i)); + CInputCoin coin = CInputCoin(pcoin, i); - if (n == nTargetValue) + if (coin.txout.nValue == nTargetValue) { - setCoinsRet.insert(coin.second); - nValueRet += coin.first; + setCoinsRet.insert(coin); + nValueRet += coin.txout.nValue; return true; } - else if (n < nTargetValue + MIN_CHANGE) + else if (coin.txout.nValue < nTargetValue + MIN_CHANGE) { vValue.push_back(coin); - nTotalLower += n; + nTotalLower += coin.txout.nValue; } - else if (n < coinLowestLarger.first) + else if (!coinLowestLarger || coin.txout.nValue < coinLowestLarger->txout.nValue) { coinLowestLarger = coin; } @@ -2123,18 +2167,18 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin { for (unsigned int i = 0; i < vValue.size(); ++i) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(vValue[i]); + nValueRet += vValue[i].txout.nValue; } return true; } if (nTotalLower < nTargetValue) { - if (coinLowestLarger.second.first == NULL) + if (!coinLowestLarger) return false; - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; return true; } @@ -2150,31 +2194,35 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin // If we have a bigger coin and (either the stochastic approximation didn't find a good solution, // or the next bigger coin is closer), return the bigger coin - if (coinLowestLarger.second.first && - ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger.first <= nBest)) + if (coinLowestLarger && + ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger->txout.nValue <= nBest)) { - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; } else { for (unsigned int i = 0; i < vValue.size(); i++) if (vfBest[i]) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(vValue[i]); + nValueRet += vValue[i].txout.nValue; } - LogPrint("selectcoins", "SelectCoins() best subset: "); - for (unsigned int i = 0; i < vValue.size(); i++) - if (vfBest[i]) - LogPrint("selectcoins", "%s ", FormatMoney(vValue[i].first)); - LogPrint("selectcoins", "total %s\n", FormatMoney(nBest)); + if (LogAcceptCategory(BCLog::SELECTCOINS)) { + LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); + for (unsigned int i = 0; i < vValue.size(); i++) { + if (vfBest[i]) { + LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(vValue[i].txout.nValue)); + } + } + LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest)); + } } return true; } -bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const +bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const { std::vector<COutput> vCoins(vAvailableCoins); @@ -2186,13 +2234,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm if (!out.fSpendable) continue; nValueRet += out.tx->tx->vout[out.i].nValue; - setCoinsRet.insert(std::make_pair(out.tx, out.i)); + setCoinsRet.insert(CInputCoin(out.tx, out.i)); } return (nValueRet >= nTargetValue); } // calculate value from preset inputs and store them - std::set<std::pair<const CWalletTx*, uint32_t> > setPresetCoins; + std::set<CInputCoin> setPresetCoins; CAmount nValueFromPresetInputs = 0; std::vector<COutPoint> vPresetInputs; @@ -2208,7 +2256,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm if (pcoin->tx->vout.size() <= outpoint.n) return false; nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(std::make_pair(pcoin, outpoint.n)); + setPresetCoins.insert(CInputCoin(pcoin, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } @@ -2216,7 +2264,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // remove preset inputs from vCoins for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coinControl && coinControl->HasSelected();) { - if (setPresetCoins.count(std::make_pair(it->tx, it->i))) + if (setPresetCoins.count(CInputCoin(it->tx, it->i))) it = vCoins.erase(it); else ++it; @@ -2243,6 +2291,28 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm return res; } +bool CWallet::SignTransaction(CMutableTransaction &tx) +{ + // sign the new tx + CTransaction txNewConst(tx); + int nIn = 0; + for (auto& input : tx.vin) { + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(input.prevout.hash); + if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { + return false; + } + const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; + const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; + SignatureData sigdata; + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { + return false; + } + UpdateTransaction(tx, nIn, sigdata); + nIn++; + } + return true; +} + bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, bool keepReserveKey, const CTxDestination& destChange) { std::vector<CRecipient> vecSend; @@ -2360,7 +2430,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT assert(txNew.nLockTime < LOCKTIME_THRESHOLD); { - std::set<std::pair<const CWalletTx*,unsigned int> > setCoins; + std::set<CInputCoin> setCoins; LOCK2(cs_main, cs_wallet); { std::vector<COutput> vAvailableCoins; @@ -2445,7 +2515,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // Reserve a new key pair from key pool CPubKey vchPubKey; bool ret; - ret = reservekey.GetReservedKey(vchPubKey); + ret = reservekey.GetReservedKey(vchPubKey, true); if (!ret) { strFailReason = _("Keypool ran out, please call keypoolrefill first"); @@ -2519,7 +2589,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // behavior." bool rbf = coinControl ? coinControl->signalRbf : fWalletRbf; for (const auto& coin : setCoins) - txNew.vin.push_back(CTxIn(coin.first->GetHash(),coin.second,CScript(), + txNew.vin.push_back(CTxIn(coin.outpoint,CScript(), std::numeric_limits<unsigned int>::max() - (rbf ? 2 : 1))); // Fill in dummy signatures for fee calculation. @@ -2543,7 +2613,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT if (coinControl && coinControl->nConfirmTarget > 0) currentConfirmationTarget = coinControl->nConfirmTarget; - CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } @@ -2602,10 +2672,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT int nIn = 0; for (const auto& coin : setCoins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->tx->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata)) + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed"); return false; @@ -2717,19 +2787,19 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator) { // payTxFee is the user-set global for desired feerate - return GetMinimumFee(nTxBytes, nConfirmTarget, pool, payTxFee.GetFee(nTxBytes)); + return GetMinimumFee(nTxBytes, nConfirmTarget, pool, estimator, payTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, CAmount targetFee) { CAmount nFeeNeeded = targetFee; // User didn't set: use -txconfirmtarget to estimate... if (nFeeNeeded == 0) { int estimateFoundTarget = nConfirmTarget; - nFeeNeeded = pool.estimateSmartFee(nConfirmTarget, &estimateFoundTarget).GetFee(nTxBytes); + nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = fallbackFee.GetFee(nTxBytes); @@ -2896,21 +2966,37 @@ bool CWallet::NewKeyPool() walletdb.ErasePool(nIndex); setKeyPool.clear(); - if (IsLocked()) + if (!TopUpKeyPool()) { return false; - - int64_t nKeys = std::max(GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t)0); - for (int i = 0; i < nKeys; i++) - { - int64_t nIndex = i+1; - walletdb.WritePool(nIndex, CKeyPool(GenerateNewKey())); - setKeyPool.insert(nIndex); } - LogPrintf("CWallet::NewKeyPool wrote %d new keys\n", nKeys); + LogPrintf("CWallet::NewKeyPool rewrote keypool\n"); } return true; } +size_t CWallet::KeypoolCountExternalKeys() +{ + AssertLockHeld(cs_wallet); // setKeyPool + + // immediately return setKeyPool's size if HD or HD_SPLIT is disabled or not supported + if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) + return setKeyPool.size(); + + CWalletDB walletdb(strWalletFile); + + // count amount of external keys + size_t amountE = 0; + for(const int64_t& id : setKeyPool) + { + CKeyPool tmpKeypool; + if (!walletdb.ReadPool(id, tmpKeypool)) + throw std::runtime_error(std::string(__func__) + ": read failed"); + amountE += !tmpKeypool.fInternal; + } + + return amountE; +} + bool CWallet::TopUpKeyPool(unsigned int kpSize) { { @@ -2919,8 +3005,6 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) if (IsLocked()) return false; - CWalletDB walletdb(strWalletFile); - // Top up key pool unsigned int nTargetSize; if (kpSize > 0) @@ -2928,21 +3012,37 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) else nTargetSize = std::max(GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); - while (setKeyPool.size() < (nTargetSize + 1)) + // count amount of available keys (internal, external) + // make sure the keypool of external and internal keys fits the user selected target (-keypool) + int64_t amountExternal = KeypoolCountExternalKeys(); + int64_t amountInternal = setKeyPool.size() - amountExternal; + int64_t missingExternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - amountExternal, (int64_t) 0); + int64_t missingInternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - amountInternal, (int64_t) 0); + + if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) + { + // don't create extra internal keys + missingInternal = 0; + } + bool internal = false; + CWalletDB walletdb(strWalletFile); + for (int64_t i = missingInternal + missingExternal; i--;) { int64_t nEnd = 1; + if (i < missingInternal) + internal = true; if (!setKeyPool.empty()) nEnd = *(--setKeyPool.end()) + 1; - if (!walletdb.WritePool(nEnd, CKeyPool(GenerateNewKey()))) + if (!walletdb.WritePool(nEnd, CKeyPool(GenerateNewKey(internal), internal))) throw std::runtime_error(std::string(__func__) + ": writing generated key failed"); setKeyPool.insert(nEnd); - LogPrintf("keypool added key %d, size=%u\n", nEnd, setKeyPool.size()); + LogPrintf("keypool added key %d, size=%u, internal=%d\n", nEnd, setKeyPool.size(), internal); } } return true; } -void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool) +void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool internal) { nIndex = -1; keypool.vchPubKey = CPubKey(); @@ -2958,14 +3058,24 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool) CWalletDB walletdb(strWalletFile); - nIndex = *(setKeyPool.begin()); - setKeyPool.erase(setKeyPool.begin()); - if (!walletdb.ReadPool(nIndex, keypool)) - throw std::runtime_error(std::string(__func__) + ": read failed"); - if (!HaveKey(keypool.vchPubKey.GetID())) - throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); - assert(keypool.vchPubKey.IsValid()); - LogPrintf("keypool reserve %d\n", nIndex); + // try to find a key that matches the internal/external filter + for(const int64_t& id : setKeyPool) + { + CKeyPool tmpKeypool; + if (!walletdb.ReadPool(id, tmpKeypool)) + throw std::runtime_error(std::string(__func__) + ": read failed"); + if (!HaveKey(tmpKeypool.vchPubKey.GetID())) + throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); + if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT) || tmpKeypool.fInternal == internal) + { + nIndex = id; + keypool = tmpKeypool; + setKeyPool.erase(id); + assert(keypool.vchPubKey.IsValid()); + LogPrintf("keypool reserve %d\n", nIndex); + return; + } + } } } @@ -2990,17 +3100,17 @@ void CWallet::ReturnKey(int64_t nIndex) LogPrintf("keypool return %d\n", nIndex); } -bool CWallet::GetKeyFromPool(CPubKey& result) +bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) { int64_t nIndex = 0; CKeyPool keypool; { LOCK(cs_wallet); - ReserveKeyFromKeyPool(nIndex, keypool); + ReserveKeyFromKeyPool(nIndex, keypool, internal); if (nIndex == -1) { if (IsLocked()) return false; - result = GenerateNewKey(); + result = GenerateNewKey(internal); return true; } KeepKey(nIndex); @@ -3017,9 +3127,33 @@ int64_t CWallet::GetOldestKeyPoolTime() if (setKeyPool.empty()) return GetTime(); - // load oldest key from keypool, get time and return CKeyPool keypool; CWalletDB walletdb(strWalletFile); + + if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) + { + // if HD & HD Chain Split is enabled, response max(oldest-internal-key, oldest-external-key) + int64_t now = GetTime(); + int64_t oldest_external = now, oldest_internal = now; + + for(const int64_t& id : setKeyPool) + { + if (!walletdb.ReadPool(id, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read failed"); + } + if (keypool.fInternal && keypool.nTime < oldest_internal) { + oldest_internal = keypool.nTime; + } + else if (!keypool.fInternal && keypool.nTime < oldest_external) { + oldest_external = keypool.nTime; + } + if (oldest_internal != now && oldest_external != now) { + break; + } + } + return std::max(oldest_internal, oldest_external); + } + // load oldest key from keypool, get time and return int64_t nIndex = *(setKeyPool.begin()); if (!walletdb.ReadPool(nIndex, keypool)) throw std::runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); @@ -3205,12 +3339,12 @@ std::set<CTxDestination> CWallet::GetAccountAddresses(const std::string& strAcco return result; } -bool CReserveKey::GetReservedKey(CPubKey& pubkey) +bool CReserveKey::GetReservedKey(CPubKey& pubkey, bool internal) { if (nIndex == -1) { CKeyPool keypool; - pwallet->ReserveKeyFromKeyPool(nIndex, keypool); + pwallet->ReserveKeyFromKeyPool(nIndex, keypool, internal); if (nIndex != -1) vchPubKey = keypool.vchPubKey; else { @@ -3258,20 +3392,9 @@ void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const } } -void CWallet::UpdatedTransaction(const uint256 &hashTx) +void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) { - { - LOCK(cs_wallet); - // Only notify UI if this transaction is in this wallet - std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(hashTx); - if (mi != mapWallet.end()) - NotifyTransactionChanged(this, hashTx, CT_UPDATED); - } -} - -void CWallet::GetScriptForMining(boost::shared_ptr<CReserveScript> &script) -{ - boost::shared_ptr<CReserveKey> rKey(new CReserveKey(this)); + std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); CPubKey pubkey; if (!rKey->GetReservedKey(pubkey)) return; @@ -3623,13 +3746,17 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) { // Create new keyUser and set as default key if (GetBoolArg("-usehd", DEFAULT_USE_HD_WALLET) && !walletInstance->IsHDEnabled()) { + + // ensure this wallet.dat can only be opened by clients supporting HD with chain split + walletInstance->SetMinVersion(FEATURE_HD_SPLIT); + // generate a new master key CPubKey masterPubKey = walletInstance->GenerateNewHDMasterKey(); if (!walletInstance->SetHDMasterKey(masterPubKey)) throw std::runtime_error(std::string(__func__) + ": Storing master key failed"); } CPubKey newDefaultKey; - if (walletInstance->GetKeyFromPool(newDefaultKey)) { + if (walletInstance->GetKeyFromPool(newDefaultKey, false)) { walletInstance->SetDefaultKey(newDefaultKey); if (!walletInstance->SetAddressBook(walletInstance->vchDefaultKey.GetID(), "", "receive")) { InitError(_("Cannot write default address") += "\n"); @@ -3735,7 +3862,7 @@ bool CWallet::InitLoadWallet() std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - if (walletFile.find_first_of("/\\") != std::string::npos) { + if (boost::filesystem::path(walletFile).filename() != walletFile) { return InitError(_("-wallet parameter must only specify a filename (not a path)")); } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { return InitError(_("Invalid characters in -wallet filename")); @@ -3865,16 +3992,16 @@ bool CWallet::BackupWallet(const std::string& strDest) bitdb.mapFileUseCount.erase(strWalletFile); // Copy wallet file - boost::filesystem::path pathSrc = GetDataDir() / strWalletFile; - boost::filesystem::path pathDest(strDest); - if (boost::filesystem::is_directory(pathDest)) + fs::path pathSrc = GetDataDir() / strWalletFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) pathDest /= strWalletFile; try { - boost::filesystem::copy_file(pathSrc, pathDest, boost::filesystem::copy_option::overwrite_if_exists); + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strWalletFile, pathDest.string()); return true; - } catch (const boost::filesystem::filesystem_error& e) { + } catch (const fs::filesystem_error& e) { LogPrintf("error copying %s to %s - %s\n", strWalletFile, pathDest.string(), e.what()); return false; } @@ -3888,12 +4015,14 @@ bool CWallet::BackupWallet(const std::string& strDest) CKeyPool::CKeyPool() { nTime = GetTime(); + fInternal = false; } -CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn) +CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) { nTime = GetTime(); vchPubKey = vchPubKeyIn; + fInternal = internalIn; } CWalletKey::CWalletKey(int64_t nExpires) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index ae4321eef8..c0ed44377f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -28,8 +28,6 @@ #include <utility> #include <vector> -#include <boost/shared_ptr.hpp> - extern CWallet* pwalletMain; /** @@ -75,6 +73,7 @@ class CReserveKey; class CScript; class CScheduler; class CTxMemPool; +class CBlockPolicyEstimator; class CWalletTx; /** (client) version numbers for particular wallet features */ @@ -86,6 +85,9 @@ enum WalletFeature FEATURE_COMPRPUBKEY = 60000, // compressed public keys FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) + + FEATURE_HD_SPLIT = 139900, // Wallet with HD chain split (change outputs will use m/0'/1'/k) + FEATURE_LATEST = FEATURE_COMPRPUBKEY // HD is optional, use FEATURE_COMPRPUBKEY as latest version }; @@ -96,9 +98,10 @@ class CKeyPool public: int64_t nTime; CPubKey vchPubKey; + bool fInternal; // for change outputs CKeyPool(); - CKeyPool(const CPubKey& vchPubKeyIn); + CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); ADD_SERIALIZE_METHODS; @@ -109,6 +112,19 @@ public: READWRITE(nVersion); READWRITE(nTime); READWRITE(vchPubKey); + if (ser_action.ForRead()) { + try { + READWRITE(fInternal); + } + catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; + } + } + else { + READWRITE(fInternal); + } } }; @@ -458,7 +474,34 @@ public: }; +class CInputCoin { +public: + CInputCoin(const CWalletTx* walletTx, unsigned int i) + { + if (!walletTx) + throw std::invalid_argument("walletTx should not be null"); + if (i >= walletTx->tx->vout.size()) + throw std::out_of_range("The output index is out of range"); + + outpoint = COutPoint(walletTx->GetHash(), i); + txout = walletTx->tx->vout[i]; + } + + COutPoint outpoint; + CTxOut txout; + bool operator<(const CInputCoin& rhs) const { + return outpoint < rhs.outpoint; + } + + bool operator!=(const CInputCoin& rhs) const { + return outpoint != rhs.outpoint; + } + + bool operator==(const CInputCoin& rhs) const { + return outpoint == rhs.outpoint; + } +}; class COutput { @@ -609,13 +652,15 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface { private: static std::atomic<bool> fFlushScheduled; + std::atomic<bool> fAbortRescan; + std::atomic<bool> fScanningWallet; /** * Select a set of coins such that nValueRet >= nTargetValue and at least * all coins from coinControl are selected; Never select unconfirmed coins * if they are not ours */ - bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; + bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; CWalletDB *pwalletdbEncryption; @@ -644,9 +689,16 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>); + /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected. + * Should be called with pindexBlock and posInBlock if this is for a transaction that is included in a block. */ + void SyncTransaction(const CTransactionRef& tx, const CBlockIndex *pindex = NULL, int posInBlock = 0); + /* the HD chain data model (external chain counters) */ CHDChain hdChain; + /* HD derive new child key (on internal or external chain) */ + void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal = false); + bool fFileBacked; std::set<int64_t> setKeyPool; @@ -726,6 +778,8 @@ public: nTimeFirstKey = 0; fBroadcastTransactions = false; nRelockTime = 0; + fAbortRescan = false; + fScanningWallet = false; } std::map<uint256, CWalletTx> mapWallet; @@ -760,7 +814,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const; + bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; @@ -770,12 +824,18 @@ public: void UnlockAllCoins(); void ListLockedCoins(std::vector<COutPoint>& vOutpts); + /* + * Rescan abort properties + */ + void AbortRescan() { fAbortRescan = true; } + bool IsAbortingRescan() { return fAbortRescan; } + bool IsScanning() { return fScanningWallet; } + /** * keystore implementation * Generate a new key */ - CPubKey GenerateNewKey(); - void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret); + CPubKey GenerateNewKey(bool internal = false); //! Adds a key to the store, and saves it to disk. bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; //! Adds a key to the store, without saving it to disk (used by LoadWallet) @@ -830,8 +890,10 @@ public: void MarkDirty(); bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); bool LoadToWallet(const CWalletTx& wtxIn); - void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock) override; - bool AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); + void TransactionAddedToMempool(const CTransactionRef& tx) override; + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; @@ -848,6 +910,7 @@ public: * calling CreateTransaction(); */ bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, bool keepReserveKey = true, const CTxDestination& destChange = CNoDestination()); + bool SignTransaction(CMutableTransaction& tx); /** * Create a new transaction paying the recipients with a set of coins @@ -862,7 +925,7 @@ public: bool AddAccountingEntry(const CAccountingEntry&); bool AddAccountingEntry(const CAccountingEntry&, CWalletDB *pwalletdb); template <typename ContainerType> - bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins); + bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const; static CFeeRate minTxFee; static CFeeRate fallbackFee; @@ -870,12 +933,12 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator); /** * Estimate the minimum fee considering required fee and targetFee or if 0 * then fee estimation for nConfirmTarget */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, CAmount targetFee); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -883,11 +946,12 @@ public: static CAmount GetRequiredFee(unsigned int nTxBytes); bool NewKeyPool(); + size_t KeypoolCountExternalKeys(); bool TopUpKeyPool(unsigned int kpSize = 0); - void ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool); + void ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool internal); void KeepKey(int64_t nIndex); void ReturnKey(int64_t nIndex); - bool GetKeyFromPool(CPubKey &key); + bool GetKeyFromPool(CPubKey &key, bool internal = false); int64_t GetOldestKeyPoolTime(); void GetAllReserveKeys(std::set<CKeyID>& setAddress) const; @@ -926,8 +990,6 @@ public: bool DelAddressBook(const CTxDestination& address); - void UpdatedTransaction(const uint256 &hashTx) override; - void Inventory(const uint256 &hash) override { { @@ -938,12 +1000,7 @@ public: } } - void GetScriptForMining(boost::shared_ptr<CReserveScript> &script) override; - void ResetRequestCount(const uint256 &hash) override - { - LOCK(cs_wallet); - mapRequestCount[hash] = 0; - }; + void GetScriptForMining(std::shared_ptr<CReserveScript> &script) override; unsigned int GetKeyPoolSize() { @@ -1035,8 +1092,10 @@ public: /* Generates a new HD master key (will not be activated) */ CPubKey GenerateNewHDMasterKey(); - /* Set the current HD master key (will reset the chain child index counters) */ - bool SetHDMasterKey(const CPubKey& key); + /* Set the current HD master key (will reset the chain child index counters) + If possibleOldChain is provided, the parameters from the old chain (version) + will be preserved. */ + bool SetHDMasterKey(const CPubKey& key, CHDChain *possibleOldChain = nullptr); }; /** A key allocated from the key pool. */ @@ -1063,7 +1122,7 @@ public: } void ReturnKey(); - bool GetReservedKey(CPubKey &pubkey); + bool GetReservedKey(CPubKey &pubkey, bool internal = false); void KeepKey(); void KeepScript() { KeepKey(); } }; @@ -1103,13 +1162,13 @@ public: // ContainerType is meant to hold pair<CWalletTx *, int>, and be iterable // so that each entry corresponds to each vIn, in order. template <typename ContainerType> -bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) +bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const { // Fill in dummy signatures for fee calculation. int nIn = 0; for (const auto& coin : coins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; if (!ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata)) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index d017965385..ceff2d36e3 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -7,6 +7,7 @@ #include "base58.h" #include "consensus/validation.h" +#include "fs.h" #include "validation.h" // For CheckTransaction #include "protocol.h" #include "serialize.h" @@ -18,7 +19,6 @@ #include <atomic> #include <boost/version.hpp> -#include <boost/filesystem.hpp> #include <boost/foreach.hpp> #include <boost/thread.hpp> @@ -745,7 +745,7 @@ DBErrors CWalletDB::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<uin } else if ((*it) == hash) { if(!EraseTx(hash)) { - LogPrint("db", "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); + LogPrint(BCLog::DB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); delerror = true; } vTxHashOut.push_back(hash); @@ -842,12 +842,12 @@ bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDa return true; } -bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr) +bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) { return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); } -bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr) +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr) { return CDB::VerifyDatabaseFile(walletFile, dataDir, errorStr, warningStr, CWalletDB::Recover); } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 4d7dfb727e..b94f341b2e 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -46,9 +46,12 @@ class CHDChain { public: uint32_t nExternalChainCounter; + uint32_t nInternalChainCounter; CKeyID masterKeyID; //!< master key hash160 - static const int CURRENT_VERSION = 1; + static const int VERSION_HD_BASE = 1; + static const int VERSION_HD_CHAIN_SPLIT = 2; + static const int CURRENT_VERSION = VERSION_HD_CHAIN_SPLIT; int nVersion; CHDChain() { SetNull(); } @@ -59,12 +62,15 @@ public: READWRITE(this->nVersion); READWRITE(nExternalChainCounter); READWRITE(masterKeyID); + if (this->nVersion >= VERSION_HD_CHAIN_SPLIT) + READWRITE(nInternalChainCounter); } void SetNull() { nVersion = CHDChain::CURRENT_VERSION; nExternalChainCounter = 0; + nInternalChainCounter = 0; masterKeyID.SetNull(); } }; @@ -179,9 +185,9 @@ public: /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 431d8c9ac9..c063898056 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -12,7 +12,7 @@ void zmqError(const char *str) { - LogPrint("zmq", "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); + LogPrint(BCLog::ZMQ, "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); } CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(NULL) @@ -72,7 +72,7 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() // Called at startup to conditionally set up ZMQ socket(s) bool CZMQNotificationInterface::Initialize() { - LogPrint("zmq", "zmq: Initialize notification interface\n"); + LogPrint(BCLog::ZMQ, "zmq: Initialize notification interface\n"); assert(!pcontext); pcontext = zmq_init(1); @@ -89,11 +89,11 @@ bool CZMQNotificationInterface::Initialize() CZMQAbstractNotifier *notifier = *i; if (notifier->Initialize(pcontext)) { - LogPrint("zmq", " Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + LogPrint(BCLog::ZMQ, " Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); } else { - LogPrint("zmq", " Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + LogPrint(BCLog::ZMQ, " Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); break; } } @@ -109,13 +109,13 @@ bool CZMQNotificationInterface::Initialize() // Called during shutdown sequence void CZMQNotificationInterface::Shutdown() { - LogPrint("zmq", "zmq: Shutdown notification interface\n"); + LogPrint(BCLog::ZMQ, "zmq: Shutdown notification interface\n"); if (pcontext) { for (std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); i!=notifiers.end(); ++i) { CZMQAbstractNotifier *notifier = *i; - LogPrint("zmq", " Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); + LogPrint(BCLog::ZMQ, " Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); notifier->Shutdown(); } zmq_ctx_destroy(pcontext); @@ -144,8 +144,12 @@ void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindexNew, co } } -void CZMQNotificationInterface::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int posInBlock) +void CZMQNotificationInterface::TransactionAddedToMempool(const CTransactionRef& ptx) { + // Used by BlockConnected and BlockDisconnected as well, because they're + // all the same external callback. + const CTransaction& tx = *ptx; + for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) { CZMQAbstractNotifier *notifier = *i; @@ -160,3 +164,19 @@ void CZMQNotificationInterface::SyncTransaction(const CTransaction& tx, const CB } } } + +void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) +{ + for (const CTransactionRef& ptx : pblock->vtx) { + // Do a normal notify for each transaction added in the block + TransactionAddedToMempool(ptx); + } +} + +void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) +{ + for (const CTransactionRef& ptx : pblock->vtx) { + // Do a normal notify for each transaction removed in block disconnection + TransactionAddedToMempool(ptx); + } +} diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index beabb78da6..eec6f7bc64 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -8,6 +8,7 @@ #include "validationinterface.h" #include <string> #include <map> +#include <list> class CBlockIndex; class CZMQAbstractNotifier; @@ -24,8 +25,10 @@ protected: void Shutdown(); // CValidationInterface - void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock); - void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); + void TransactionAddedToMempool(const CTransactionRef& tx) override; + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; private: CZMQNotificationInterface(); diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 4e13cbcaf4..700c39f66e 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "chain.h" #include "chainparams.h" #include "streams.h" #include "zmqpublishnotifier.h" @@ -89,7 +90,7 @@ bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) } else { - LogPrint("zmq", "zmq: Reusing socket for address %s\n", address); + LogPrint(BCLog::ZMQ, "zmq: Reusing socket for address %s\n", address); psocket = i->second->psocket; mapPublishNotifiers.insert(std::make_pair(address, this)); @@ -119,7 +120,7 @@ void CZMQAbstractPublishNotifier::Shutdown() if (count == 1) { - LogPrint("zmq", "Close socket at address %s\n", address); + LogPrint(BCLog::ZMQ, "Close socket at address %s\n", address); int linger = 0; zmq_setsockopt(psocket, ZMQ_LINGER, &linger, sizeof(linger)); zmq_close(psocket); @@ -148,7 +149,7 @@ bool CZMQAbstractPublishNotifier::SendMessage(const char *command, const void* d bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); - LogPrint("zmq", "zmq: Publish hashblock %s\n", hash.GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish hashblock %s\n", hash.GetHex()); char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; @@ -158,7 +159,7 @@ bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &transaction) { uint256 hash = transaction.GetHash(); - LogPrint("zmq", "zmq: Publish hashtx %s\n", hash.GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish hashtx %s\n", hash.GetHex()); char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; @@ -167,7 +168,7 @@ bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &t bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { - LogPrint("zmq", "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish rawblock %s\n", pindex->GetBlockHash().GetHex()); const Consensus::Params& consensusParams = Params().GetConsensus(); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); @@ -189,7 +190,7 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &transaction) { uint256 hash = transaction.GetHash(); - LogPrint("zmq", "zmq: Publish rawtx %s\n", hash.GetHex()); + LogPrint(BCLog::ZMQ, "zmq: Publish rawtx %s\n", hash.GetHex()); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ss << transaction; return SendMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); |