diff options
Diffstat (limited to 'src')
138 files changed, 4621 insertions, 2414 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 401705bafa..ae2eb29c94 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -117,6 +117,7 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ noui.h \ + policy/feerate.h \ policy/fees.h \ policy/policy.h \ policy/rbf.h \ @@ -158,6 +159,7 @@ BITCOIN_CORE_H = \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ + wallet/feebumper.h \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ @@ -178,8 +180,8 @@ libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ - addrman.cpp \ addrdb.cpp \ + addrman.cpp \ bloom.cpp \ blockencodings.cpp \ chain.cpp \ @@ -233,6 +235,7 @@ libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/crypter.cpp \ wallet/db.cpp \ + wallet/feebumper.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ @@ -246,6 +249,8 @@ crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ + crypto/chacha20.h \ + crypto/chacha20.cpp \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ @@ -299,7 +304,6 @@ libbitcoin_consensus_a_SOURCES = \ libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ - amount.cpp \ base58.cpp \ chainparams.cpp \ coins.cpp \ @@ -310,6 +314,7 @@ libbitcoin_common_a_SOURCES = \ keystore.cpp \ netaddress.cpp \ netbase.cpp \ + policy/feerate.cpp \ protocol.cpp \ scheduler.cpp \ script/sign.cpp \ @@ -462,6 +467,14 @@ DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) + +config/bitcoin-config.h: config/stamp-h1 + @$(MAKE) -C $(top_builddir) $(subdir)/$(@) +config/stamp-h1: $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in $(top_builddir)/config.status + $(AM_V_at)$(MAKE) -C $(top_builddir) $(subdir)/$(@) +$(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) + $(AM_V_at)$(MAKE) -C $(top_srcdir) $(subdir)/config/bitcoin-config.h.in + clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean diff --git a/src/Makefile.test.include b/src/Makefile.test.include index fe0ed59fe2..ee1c11ff1f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -57,8 +57,8 @@ BITCOIN_TESTS =\ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ - test/random_tests.cpp \ test/raii_event_tests.cpp \ + test/random_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ @@ -78,6 +78,7 @@ BITCOIN_TESTS =\ test/testutil.cpp \ test/testutil.h \ test/timedata_tests.cpp \ + test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ @@ -97,7 +98,7 @@ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) + $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) diff --git a/src/addrman.cpp b/src/addrman.cpp index ed9c128eb2..4a408b9beb 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -351,8 +351,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.rand32()) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; + nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); @@ -368,8 +368,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.rand32()) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; + nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); diff --git a/src/addrman.h b/src/addrman.h index f123b20b1b..70d907488f 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -136,13 +136,13 @@ public: */ //! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT 256 +#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 //! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT 1024 +#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 //! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE 64 +#define ADDRMAN_BUCKET_SIZE_LOG2 6 //! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 @@ -171,6 +171,11 @@ public: //! the maximum number of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX 2500 +//! Convenience +#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) +#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) +#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) + /** * Stochastical (IP) address manager */ diff --git a/src/amount.h b/src/amount.h index 93060f7193..2bd367cba2 100644 --- a/src/amount.h +++ b/src/amount.h @@ -6,10 +6,7 @@ #ifndef BITCOIN_AMOUNT_H #define BITCOIN_AMOUNT_H -#include "serialize.h" - -#include <stdlib.h> -#include <string> +#include <stdint.h> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; @@ -17,8 +14,6 @@ typedef int64_t CAmount; static const CAmount COIN = 100000000; static const CAmount CENT = 1000000; -extern const std::string CURRENCY_UNIT; - /** No amount larger than this (in satoshi) is valid. * * Note that this constant is *not* the total money supply, which in Bitcoin @@ -31,42 +26,4 @@ extern const std::string CURRENCY_UNIT; static const CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -/** - * Fee rate in satoshis per kilobyte: CAmount / kB - */ -class CFeeRate -{ -private: - CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes -public: - /** Fee rate of 0 satoshis per kB */ - CFeeRate() : nSatoshisPerK(0) { } - explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } - /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ - CFeeRate(const CAmount& nFeePaid, size_t nBytes); - CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } - /** - * Return the fee in satoshis for the given size in bytes. - */ - CAmount GetFee(size_t nBytes) const; - /** - * Return the fee in satoshis for a size of 1000 bytes - */ - CAmount GetFeePerK() const { return GetFee(1000); } - friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } - friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } - friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } - friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } - friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } - CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } - std::string ToString() const; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } -}; - #endif // BITCOIN_AMOUNT_H diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index b0df3d2b04..33631d2d15 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,6 +5,7 @@ #include "bench.h" #include "perf.h" +#include <assert.h> #include <iostream> #include <iomanip> #include <sys/time.h> diff --git a/src/bench/bench.h b/src/bench/bench.h index f12a41126c..1f36f2a4bc 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <functional> +#include <limits> #include <map> #include <string> -#include <boost/function.hpp> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> @@ -59,7 +60,7 @@ namespace benchmark { bool KeepRunning(); }; - typedef boost::function<void(State&)> BenchFunction; + typedef std::function<void(State&)> BenchFunction; class BenchRunner { diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 230e4ca773..195388839e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -22,7 +22,7 @@ static void DeserializeBlockTest(benchmark::State& state) CDataStream stream((const char*)block_bench::block413567, (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], SER_NETWORK, PROTOCOL_VERSION); - char a; + char a = '\0'; stream.write(&a, 1); // Prevent compaction while (state.KeepRunning()) { @@ -37,10 +37,10 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) CDataStream stream((const char*)block_bench::block413567, (const char*)&block_bench::block413567[sizeof(block_bench::block413567)], SER_NETWORK, PROTOCOL_VERSION); - char a; + char a = '\0'; stream.write(&a, 1); // Prevent compaction - Consensus::Params params = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); while (state.KeepRunning()) { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here @@ -48,7 +48,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) assert(stream.Rewind(sizeof(block_bench::block413567))); CValidationState validationState; - assert(CheckBlock(block, validationState, params)); + assert(CheckBlock(block, validationState, chainParams->GetConsensus())); } } diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 6fa9fe4fe8..88a2a570f9 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -68,7 +68,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) PrevectorJob(){ } PrevectorJob(FastRandomContext& insecure_rand){ - p.resize(insecure_rand.rand32() % (PREVECTOR_SIZE*2)); + p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } bool operator()() { diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 06882f1514..42891f345b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -48,7 +48,7 @@ static void CoinSelection(benchmark::State& state) addCoin(1000 * COIN, wallet, vCoins); addCoin(3 * COIN, wallet, vCoins); - std::set<std::pair<const CWalletTx*, unsigned int> > setCoinsRet; + std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool success = wallet.SelectCoinsMinConf(1003 * COIN, 1, 6, 0, vCoins, setCoinsRet, nValueRet); assert(success); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 737d3572ae..2914a36c7b 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -7,6 +7,7 @@ #include "bench.h" #include "bloom.h" #include "hash.h" +#include "random.h" #include "uint256.h" #include "utiltime.h" #include "crypto/ripemd160.h" @@ -69,6 +70,28 @@ static void SipHash_32b(benchmark::State& state) } } +static void FastRandom_32bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.rand32(); + } + } +} + +static void FastRandom_1bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.randbool(); + } + } +} + BENCHMARK(RIPEMD160); BENCHMARK(SHA1); BENCHMARK(SHA256); @@ -76,3 +99,5 @@ BENCHMARK(SHA512); BENCHMARK(SHA256_32b); BENCHMARK(SipHash_32b); +BENCHMARK(FastRandom_32bit); +BENCHMARK(FastRandom_1bit); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 5edd43d41e..885b787b4d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -30,6 +30,8 @@ static const int CONTINUE_EXECUTION=-1; std::string HelpMessageCli() { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); std::string strUsage; strUsage += HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); @@ -38,7 +40,7 @@ std::string HelpMessageCli() AppendParamsHelpMessages(strUsage); strUsage += HelpMessageOpt("-named", strprintf(_("Pass named instead of positional arguments (default: %s)"), DEFAULT_NAMED)); strUsage += HelpMessageOpt("-rpcconnect=<ip>", strprintf(_("Send commands to node running on <ip> (default: %s)"), DEFAULT_RPCCONNECT)); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 83b855cbcf..45738b5df8 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -657,11 +657,13 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, MutateTxDelOutput(tx, commandVal); else if (command == "outaddr") MutateTxAddOutAddr(tx, commandVal); - else if (command == "outpubkey") + else if (command == "outpubkey") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutPubKey(tx, commandVal); - else if (command == "outmultisig") + } else if (command == "outmultisig") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutMultiSig(tx, commandVal); - else if (command == "outscript") + } else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); else if (command == "outdata") MutateTxAddOutData(tx, commandVal); diff --git a/src/bloom.cpp b/src/bloom.cpp index ac3e565721..7ed982c984 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -19,7 +19,7 @@ #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** * The ideal size for a bloom filter with a given number of elements and false positive rate is: * - nElements * log(fp rate) / ln(2)^2 @@ -40,7 +40,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int } // Private constructor used by CRollingBloomFilter -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn) : vData((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)) / 8), isFull(false), isEmpty(true), @@ -120,7 +120,7 @@ void CBloomFilter::clear() isEmpty = true; } -void CBloomFilter::reset(unsigned int nNewTweak) +void CBloomFilter::reset(const unsigned int nNewTweak) { clear(); nTweak = nNewTweak; @@ -214,7 +214,7 @@ void CBloomFilter::UpdateEmptyFull() isEmpty = empty; } -CRollingBloomFilter::CRollingBloomFilter(unsigned int nElements, double fpRate) +CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate) { double logFpRate = log(fpRate); /* The optimal number of hash functions is log(fpRate) / log(0.5), but diff --git a/src/bloom.h b/src/bloom.h index 5ad727c330..7ca9682239 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -54,7 +54,7 @@ private: unsigned int Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const; // Private constructor for CRollingBloomFilter, no restrictions on size - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak); friend class CRollingBloomFilter; public: @@ -67,7 +67,7 @@ public: * It should generally always be a random value (and is largely only exposed for unit testing) * nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) */ - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} ADD_SERIALIZE_METHODS; @@ -89,7 +89,7 @@ public: bool contains(const uint256& hash) const; void clear(); - void reset(unsigned int nNewTweak); + void reset(const unsigned int nNewTweak); //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) @@ -122,7 +122,7 @@ public: // A random bloom filter calls GetRand() at creation time. // Don't create global CRollingBloomFilter objects, as they may be // constructed before the randomizer is properly initialized. - CRollingBloomFilter(unsigned int nElements, double nFPRate); + CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); void insert(const uint256& hash); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 1dc29826af..5055fb3e0a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -55,6 +55,12 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits return CreateGenesisBlock(pszTimestamp, genesisOutputScript, nTime, nNonce, nBits, nVersion, genesisReward); } +void CChainParams::UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +{ + consensus.vDeployments[d].nStartTime = nStartTime; + consensus.vDeployments[d].nTimeout = nTimeout; +} + /** * Main network */ @@ -165,7 +171,6 @@ public: }; } }; -static CMainParams mainParams; /** * Testnet (v3) @@ -253,7 +258,6 @@ public: } }; -static CTestNetParams testNetParams; /** * Regression test @@ -326,42 +330,34 @@ public: base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >(); base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >(); } - - void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) - { - consensus.vDeployments[d].nStartTime = nStartTime; - consensus.vDeployments[d].nTimeout = nTimeout; - } }; -static CRegTestParams regTestParams; -static CChainParams *pCurrentParams = 0; +static std::unique_ptr<CChainParams> globalChainParams; const CChainParams &Params() { - assert(pCurrentParams); - return *pCurrentParams; + assert(globalChainParams); + return *globalChainParams; } -CChainParams& Params(const std::string& chain) +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CChainParams>(new CMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CChainParams>(new CTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; - else - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + return std::unique_ptr<CChainParams>(new CRegTestParams()); + throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectParams(const std::string& network) { SelectBaseParams(network); - pCurrentParams = &Params(network); + globalChainParams = CreateChainParams(network); } -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) { - regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout); + globalChainParams->UpdateBIP9Parameters(d, nStartTime, nTimeout); } diff --git a/src/chainparams.h b/src/chainparams.h index 4fe88c691c..e5312d1080 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -11,6 +11,7 @@ #include "primitives/block.h" #include "protocol.h" +#include <memory> #include <vector> struct CDNSSeedData { @@ -75,6 +76,7 @@ public: const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } const ChainTxData& TxData() const { return chainTxData; } + void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); protected: CChainParams() {} @@ -95,15 +97,17 @@ protected: }; /** - * Return the currently selected parameters. This won't change after app - * startup, except for unit tests. + * Creates and returns a std::unique_ptr<CChainParams> of the chosen chain. + * @returns a CChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. */ -const CChainParams &Params(); +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain); /** - * @returns CChainParams for the given BIP70 chain name. + * Return the currently selected parameters. This won't change after app + * startup, except for unit tests. */ -CChainParams& Params(const std::string& chain); +const CChainParams &Params(); /** * Sets the params returned by Params() to those for the given BIP70 chain name. @@ -114,6 +118,6 @@ void SelectParams(const std::string& chain); /** * Allows modifying the BIP9 regtest parameters. */ -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); +void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index cb71a8b550..43c9a13c54 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -35,7 +35,6 @@ public: nRPCPort = 8332; } }; -static CBaseMainParams mainParams; /** * Testnet (v3) @@ -49,7 +48,6 @@ public: strDataDir = "testnet3"; } }; -static CBaseTestNetParams testNetParams; /* * Regression test @@ -63,31 +61,30 @@ public: strDataDir = "regtest"; } }; -static CBaseRegTestParams regTestParams; -static CBaseChainParams* pCurrentBaseParams = 0; +static std::unique_ptr<CBaseChainParams> globalChainBaseParams; const CBaseChainParams& BaseParams() { - assert(pCurrentBaseParams); - return *pCurrentBaseParams; + assert(globalChainBaseParams); + return *globalChainBaseParams; } -CBaseChainParams& BaseParams(const std::string& chain) +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CBaseChainParams>(new CBaseMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CBaseChainParams>(new CBaseTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; + return std::unique_ptr<CBaseChainParams>(new CBaseRegTestParams()); else throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectBaseParams(const std::string& chain) { - pCurrentBaseParams = &BaseParams(chain); + globalChainBaseParams = CreateBaseChainParams(chain); } std::string ChainNameFromCommandLine() @@ -103,8 +100,3 @@ std::string ChainNameFromCommandLine() return CBaseChainParams::TESTNET; return CBaseChainParams::MAIN; } - -bool AreBaseParamsConfigured() -{ - return pCurrentBaseParams != NULL; -} diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 59493afb9b..fc101f5b77 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CHAINPARAMSBASE_H #define BITCOIN_CHAINPARAMSBASE_H +#include <memory> #include <string> #include <vector> @@ -31,6 +32,13 @@ protected: }; /** + * Creates and returns a std::unique_ptr<CBaseChainParams> of the chosen chain. + * @returns a CBaseChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. + */ +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain); + +/** * Append the help messages for the chainparams options to the * parameter string. */ @@ -42,8 +50,6 @@ void AppendParamsHelpMessages(std::string& strUsage, bool debugHelp=true); */ const CBaseChainParams& BaseParams(); -CBaseChainParams& BaseParams(const std::string& chain); - /** Sets the params returned by Params() to those for the given network. */ void SelectBaseParams(const std::string& chain); @@ -53,10 +59,4 @@ void SelectBaseParams(const std::string& chain); */ std::string ChainNameFromCommandLine(); -/** - * Return true if SelectBaseParamsFromCommandLine() has been called to select - * a network. - */ -bool AreBaseParamsConfigured(); - #endif // BITCOIN_CHAINPARAMSBASE_H diff --git a/src/checkqueue.h b/src/checkqueue.h index ea12df66dd..63c104c02a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H +#include "sync.h" + #include <algorithm> #include <vector> diff --git a/src/clientversion.h b/src/clientversion.h index 69154d546d..3d5392619b 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -7,29 +7,13 @@ #if defined(HAVE_CONFIG_H) #include "config/bitcoin-config.h" -#else - -/** - * client versioning and copyright year - */ - -//! These need to be macros, as clientversion.cpp's and bitcoin*-res.rc's voodoo requires it -#define CLIENT_VERSION_MAJOR 0 -#define CLIENT_VERSION_MINOR 14 -#define CLIENT_VERSION_REVISION 99 -#define CLIENT_VERSION_BUILD 0 - -//! Set to true for release, false for prerelease or test build -#define CLIENT_VERSION_IS_RELEASE false - -/** - * Copyright year (2009-this) - * Todo: update this when changing our copyright comments in the source - */ -#define COPYRIGHT_YEAR 2017 - #endif //HAVE_CONFIG_H +// Check that required client information is defined +#if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_REVISION) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) +#error Client version information missing: version is not defined by bitcoin-config.h or in any other way +#endif + /** * Converts the parameter X to a string after macro replacement on X has been performed. * Don't merge these into one macro! diff --git a/src/coins.h b/src/coins.h index 8ee49b33ae..065bae56e9 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" @@ -17,7 +18,7 @@ #include <stdint.h> #include <boost/foreach.hpp> -#include <boost/unordered_map.hpp> +#include <unordered_map> /** * Pruned version of CTransaction: only retains metadata and unspent transaction outputs @@ -279,7 +280,7 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : coins(), flags(0) {} }; -typedef boost::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; +typedef std::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor diff --git a/src/core_write.cpp b/src/core_write.cpp index a3ca87c8b5..d116e617ee 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -151,6 +151,8 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry) entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); entry.pushKV("version", tx.nVersion); + entry.pushKV("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("locktime", (int64_t)tx.nLockTime); UniValue vin(UniValue::VARR); diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp new file mode 100644 index 0000000000..816ae870e1 --- /dev/null +++ b/src/crypto/chacha20.cpp @@ -0,0 +1,180 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation 'merged' by D. J. Bernstein +// See https://cr.yp.to/chacha.html. + +#include "crypto/common.h" +#include "crypto/chacha20.h" + +#include <string.h> + +constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); } + +#define QUARTERROUND(a,b,c,d) \ + a += b; d = rotl32(d ^ a, 16); \ + c += d; b = rotl32(b ^ c, 12); \ + a += b; d = rotl32(d ^ a, 8); \ + c += d; b = rotl32(b ^ c, 7); + +static const unsigned char sigma[] = "expand 32-byte k"; +static const unsigned char tau[] = "expand 16-byte k"; + +void ChaCha20::SetKey(const unsigned char* k, size_t keylen) +{ + const unsigned char *constants; + + input[4] = ReadLE32(k + 0); + input[5] = ReadLE32(k + 4); + input[6] = ReadLE32(k + 8); + input[7] = ReadLE32(k + 12); + if (keylen == 32) { /* recommended */ + k += 16; + constants = sigma; + } else { /* keylen == 16 */ + constants = tau; + } + input[8] = ReadLE32(k + 0); + input[9] = ReadLE32(k + 4); + input[10] = ReadLE32(k + 8); + input[11] = ReadLE32(k + 12); + input[0] = ReadLE32(constants + 0); + input[1] = ReadLE32(constants + 4); + input[2] = ReadLE32(constants + 8); + input[3] = ReadLE32(constants + 12); + input[12] = 0; + input[13] = 0; + input[14] = 0; + input[15] = 0; +} + +ChaCha20::ChaCha20() +{ + memset(input, 0, sizeof(input)); +} + +ChaCha20::ChaCha20(const unsigned char* k, size_t keylen) +{ + SetKey(k, keylen); +} + +void ChaCha20::SetIV(uint64_t iv) +{ + input[14] = iv; + input[15] = iv >> 32; +} + +void ChaCha20::Seek(uint64_t pos) +{ + input[12] = pos; + input[13] = pos >> 32; +} + +void ChaCha20::Output(unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = NULL; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h new file mode 100644 index 0000000000..a305977bcd --- /dev/null +++ b/src/crypto/chacha20.h @@ -0,0 +1,26 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20_H +#define BITCOIN_CRYPTO_CHACHA20_H + +#include <stdint.h> +#include <stdlib.h> + +/** A PRNG class for ChaCha20. */ +class ChaCha20 +{ +private: + uint32_t input[16]; + +public: + ChaCha20(); + ChaCha20(const unsigned char* key, size_t keylen); + void SetKey(const unsigned char* key, size_t keylen); + void SetIV(uint64_t iv); + void Seek(uint64_t pos); + void Output(unsigned char* output, size_t bytes); +}; + +#endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/common.h b/src/crypto/common.h index 4a9d1150b6..bcca3d30ea 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -79,4 +79,25 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) memcpy(ptr, (char*)&v, 8); } +/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ +uint64_t static inline CountBits(uint64_t x) +{ +#ifdef HAVE_DECL___BUILTIN_CLZL + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; + } +#endif +#ifdef HAVE_DECL___BUILTIN_CLZLL + if (sizeof(unsigned long long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; + } +#endif + int ret = 0; + while (x) { + x >>= 1; + ++ret; + } + return ret; +} + #endif // BITCOIN_CRYPTO_COMMON_H diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 21c64c5c83..5ab6d8d732 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -30,7 +30,7 @@ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; class HTTPRPCTimer : public RPCTimerBase { public: - HTTPRPCTimer(struct event_base* eventBase, boost::function<void(void)>& func, int64_t millis) : + HTTPRPCTimer(struct event_base* eventBase, std::function<void(void)>& func, int64_t millis) : ev(eventBase, false, func) { struct timeval tv; @@ -52,7 +52,7 @@ public: { return "HTTP"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new HTTPRPCTimer(base, func, millis); } @@ -93,9 +93,9 @@ static bool multiUserAuthorized(std::string strUserPass) std::string strUser = strUserPass.substr(0, strUserPass.find(":")); std::string strPass = strUserPass.substr(strUserPass.find(":") + 1); - if (mapMultiArgs.count("-rpcauth") > 0) { + if (gArgs.IsArgSet("-rpcauth")) { //Search for multi-user login/pass "rpcauth" from config - BOOST_FOREACH(std::string strRPCAuth, mapMultiArgs.at("-rpcauth")) + BOOST_FOREACH(std::string strRPCAuth, gArgs.GetArgs("-rpcauth")) { std::vector<std::string> vFields; boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 347433eb11..0d1cba3fd2 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -196,9 +196,8 @@ static bool InitHTTPAllowList() LookupHost("::1", localv6, false); rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost - if (mapMultiArgs.count("-rpcallowip")) { - const std::vector<std::string>& vAllow = mapMultiArgs.at("-rpcallowip"); - for (std::string strAllow : vAllow) { + if (gArgs.IsArgSet("-rpcallowip")) { + for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { @@ -321,12 +320,11 @@ static bool HTTPBindAddresses(struct evhttp* http) if (IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); } - } else if (mapMultiArgs.count("-rpcbind")) { // Specific bind address - const std::vector<std::string>& vbind = mapMultiArgs.at("-rpcbind"); - for (std::vector<std::string>::const_iterator i = vbind.begin(); i != vbind.end(); ++i) { + } else if (gArgs.IsArgSet("-rpcbind")) { // Specific bind address + for (const std::string& strRPCBind : gArgs.GetArgs("-rpcbind")) { int port = defaultPort; std::string host; - SplitHostPort(*i, port, host); + SplitHostPort(strRPCBind, port, host); endpoints.push_back(std::make_pair(host, port)); } } else { // No specific bind address specified, bind to any @@ -384,15 +382,13 @@ bool InitHTTPServer() // Redirect libevent's logging to our own log event_set_log_callback(&libevent_log_cb); -#if LIBEVENT_VERSION_NUMBER >= 0x02010100 - // If -debug=libevent, set full libevent debugging. - // Otherwise, disable all libevent debugging. - if (LogAcceptCategory(BCLog::LIBEVENT)) { - event_enable_debug_logging(EVENT_DBG_ALL); - } else { - event_enable_debug_logging(EVENT_DBG_NONE); + // Update libevent's log handling. Returns false if our version of + // libevent doesn't support debug logging, in which case we should + // clear the BCLog::LIBEVENT flag. + if (!UpdateHTTPServerLogging(logCategories & BCLog::LIBEVENT)) { + logCategories &= ~BCLog::LIBEVENT; } -#endif + #ifdef WIN32 evthread_use_windows_threads(); #else @@ -435,6 +431,20 @@ bool InitHTTPServer() return true; } +bool UpdateHTTPServerLogging(bool enable) { +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + if (enable) { + event_enable_debug_logging(EVENT_DBG_ALL); + } else { + event_enable_debug_logging(EVENT_DBG_NONE); + } + return true; +#else + // Can't update libevent logging if version < 02010100 + return false; +#endif +} + std::thread threadHTTP; std::future<bool> threadResult; diff --git a/src/httpserver.h b/src/httpserver.h index b55b253bea..6be9950682 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -32,6 +32,10 @@ void InterruptHTTPServer(); /** Stop HTTP server */ void StopHTTPServer(); +/** Change logging level for libevent. Removes BCLog::LIBEVENT from logCategories if + * libevent doesn't support debug logging.*/ +bool UpdateHTTPServerLogging(bool enable); + /** Handler for requests to a certain HTTP path */ typedef std::function<bool(HTTPRequest* req, const std::string &)> HTTPRequestHandler; /** Register handler for prefix. diff --git a/src/init.cpp b/src/init.cpp index 1e7e388a52..3bbdb16c3b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -25,6 +25,8 @@ #include "netbase.h" #include "net.h" #include "net_processing.h" +#include "policy/feerate.h" +#include "policy/fees.h" #include "policy/policy.h" #include "rpc/server.h" #include "rpc/register.h" @@ -57,7 +59,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/bind.hpp> -#include <boost/function.hpp> #include <boost/interprocess/sync/file_lock.hpp> #include <boost/thread.hpp> #include <openssl/crypto.h> @@ -207,15 +208,17 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); - if (fDumpMempoolLater) + if (fDumpMempoolLater && GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); + } if (fFeeEstimatesInitialized) { + ::feeEstimator.FlushUnconfirmed(::mempool); fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) - mempool.WriteFeeEstimates(est_fileout); + ::feeEstimator.Write(est_fileout); else LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, est_path.string()); fFeeEstimatesInitialized = false; @@ -326,6 +329,10 @@ void OnRPCPreCommand(const CRPCCommand& cmd) std::string HelpMessage(HelpMessageMode mode) { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); + const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); + const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); const bool showDebug = GetBoolArg("-help-debug", false); // When adding new options to the categories, please keep and ensure alphabetical ordering. @@ -337,7 +344,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-blocknotify=<cmd>", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); if (showDebug) strUsage += HelpMessageOpt("-blocksonly", strprintf(_("Whether to operate in a blocks only mode (default: %u)"), DEFAULT_BLOCKSONLY)); - strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), Params(CBaseChainParams::MAIN).GetConsensus().defaultAssumeValid.GetHex(), Params(CBaseChainParams::TESTNET).GetConsensus().defaultAssumeValid.GetHex())); + strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex())); strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); if (mode == HMM_BITCOIND) { @@ -353,6 +360,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt("-blockreconstructionextratxn=<n>", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); @@ -390,7 +398,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-onlynet=<net>", _("Only connect to nodes in network <net> (ipv4, ipv6 or onion)")); strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt("-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), DEFAULT_PEERBLOOMFILTERS)); - strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort())); + strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); @@ -427,14 +435,16 @@ std::string HelpMessage(HelpMessageMode mode) { strUsage += HelpMessageOpt("-checkblocks=<n>", strprintf(_("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); strUsage += HelpMessageOpt("-checklevel=<n>", strprintf(_("How thorough the block verification of -checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); - strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); - strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", defaultChainParams->DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", defaultChainParams->DefaultConsistencyChecks())); strUsage += HelpMessageOpt("-checkpoints", strprintf("Disable expensive verification for known chain history (default: %u)", DEFAULT_CHECKPOINTS_ENABLED)); strUsage += HelpMessageOpt("-disablesafemode", strprintf("Disable safemode, override a real safe mode event (default: %u)", DEFAULT_DISABLE_SAFEMODE)); strUsage += HelpMessageOpt("-testsafemode", strprintf("Force safe mode (default: %u)", DEFAULT_TESTSAFEMODE)); strUsage += HelpMessageOpt("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages"); strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages"); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT)); + strUsage += HelpMessageOpt("-stopatheight", strprintf("Stop running after reaching the given height in the main chain (default: %u)", DEFAULT_STOPATHEIGHT)); + strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); @@ -467,7 +477,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) { - strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !Params(CBaseChainParams::TESTNET).RequireStandard())); + strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", defaultChainParams->RequireStandard())); strUsage += HelpMessageOpt("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define cost of relay, used for mempool limiting and BIP 125 replacement. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE))); strUsage += HelpMessageOpt("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to defined dust, the value of an output such that it will cost about 1/3 of its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE))); } @@ -495,7 +505,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcauth=<userpw>", _("Username and hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcuser. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times")); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcallowip=<ip>", _("Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); @@ -676,8 +686,10 @@ void ThreadImport(std::vector<fs::path> vImportFiles) StartShutdown(); } } // End scope of CImportingNow - LoadMempool(); - fDumpMempoolLater = !fRequestShutdown; + if (GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + LoadMempool(); + fDumpMempoolLater = !fRequestShutdown; + } } /** Sanity checks @@ -734,7 +746,7 @@ void InitParameterInteraction() LogPrintf("%s: parameter interaction: -whitebind set -> setting -listen=1\n", __func__); } - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) { + if (gArgs.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen by default if (SoftSetBoolArg("-dnsseed", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -dnsseed=0\n", __func__); @@ -888,8 +900,8 @@ bool AppInitParameterInteraction() // Make sure enough file descriptors are available int nBind = std::max( - (mapMultiArgs.count("-bind") ? mapMultiArgs.at("-bind").size() : 0) + - (mapMultiArgs.count("-whitebind") ? mapMultiArgs.at("-whitebind").size() : 0), size_t(1)); + (gArgs.IsArgSet("-bind") ? gArgs.GetArgs("-bind").size() : 0) + + (gArgs.IsArgSet("-whitebind") ? gArgs.GetArgs("-whitebind").size() : 0), size_t(1)); nUserMaxConnections = GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); @@ -904,9 +916,9 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (mapMultiArgs.count("-debug") > 0) { + if (gArgs.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string>& categories = mapMultiArgs.at("-debug"); + const std::vector<std::string> categories = gArgs.GetArgs("-debug"); if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { for (const auto& cat : categories) { @@ -921,9 +933,8 @@ bool AppInitParameterInteraction() } // Now remove the logging categories which were explicitly excluded - if (mapMultiArgs.count("-debugexclude") > 0) { - const std::vector<std::string>& excludedCategories = mapMultiArgs.at("-debugexclude"); - for (const auto& cat : excludedCategories) { + if (gArgs.IsArgSet("-debugexclude")) { + for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { uint32_t flag = 0; if (!GetLogCategory(&flag, &cat)) { InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); @@ -1093,15 +1104,14 @@ bool AppInitParameterInteraction() fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } - if (mapMultiArgs.count("-bip9params")) { + if (gArgs.IsArgSet("-bip9params")) { // Allow overriding BIP9 parameters for testing if (!chainparams.MineBlocksOnDemand()) { return InitError("BIP9 parameters may only be overridden on regtest."); } - const std::vector<std::string>& deployments = mapMultiArgs.at("-bip9params"); - for (auto i : deployments) { + for (const std::string& strDeployment : gArgs.GetArgs("-bip9params")) { std::vector<std::string> vDeploymentParams; - boost::split(vDeploymentParams, i, boost::is_any_of(":")); + boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); if (vDeploymentParams.size() != 3) { return InitError("BIP9 parameters malformed, expecting deployment:start:end"); } @@ -1116,7 +1126,7 @@ bool AppInitParameterInteraction() for (int j=0; j<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[j].name) == 0) { - UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); + UpdateBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); found = true; LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); break; @@ -1247,8 +1257,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; - if (mapMultiArgs.count("-uacomment")) { - BOOST_FOREACH(std::string cmt, mapMultiArgs.at("-uacomment")) + if (gArgs.IsArgSet("-uacomment")) { + BOOST_FOREACH(std::string cmt, gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); @@ -1261,9 +1271,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } - if (mapMultiArgs.count("-onlynet")) { + if (gArgs.IsArgSet("-onlynet")) { std::set<enum Network> nets; - BOOST_FOREACH(const std::string& snet, mapMultiArgs.at("-onlynet")) { + BOOST_FOREACH(const std::string& snet, gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); @@ -1276,8 +1286,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-whitelist")) { - BOOST_FOREACH(const std::string& net, mapMultiArgs.at("-whitelist")) { + if (gArgs.IsArgSet("-whitelist")) { + BOOST_FOREACH(const std::string& net, gArgs.GetArgs("-whitelist")) { CSubNet subnet; LookupSubNet(net.c_str(), subnet); if (!subnet.IsValid()) @@ -1338,16 +1348,16 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (fListen) { bool fBound = false; - if (mapMultiArgs.count("-bind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-bind")) { + if (gArgs.IsArgSet("-bind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) return InitError(ResolveErrMsg("bind", strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR)); } } - if (mapMultiArgs.count("-whitebind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-whitebind")) { + if (gArgs.IsArgSet("-whitebind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-whitebind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, 0, false)) return InitError(ResolveErrMsg("whitebind", strBind)); @@ -1356,7 +1366,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR | BF_WHITELIST)); } } - if (!mapMultiArgs.count("-bind") && !mapMultiArgs.count("-whitebind")) { + if (!gArgs.IsArgSet("-bind") && !gArgs.IsArgSet("-whitebind")) { struct in_addr inaddr_any; inaddr_any.s_addr = INADDR_ANY; fBound |= Bind(connman, CService(in6addr_any, GetListenPort()), BF_NONE); @@ -1366,8 +1376,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) return InitError(_("Failed to listen on any port. Use -listen=0 if you want this.")); } - if (mapMultiArgs.count("-externalip")) { - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-externalip")) { + if (gArgs.IsArgSet("-externalip")) { + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); @@ -1376,8 +1386,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-seednode")) { - BOOST_FOREACH(const std::string& strDest, mapMultiArgs.at("-seednode")) + if (gArgs.IsArgSet("-seednode")) { + BOOST_FOREACH(const std::string& strDest, gArgs.GetArgs("-seednode")) connman.AddOneShot(strDest); } @@ -1550,7 +1560,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); // Allowed to fail as this file IS missing on first startup. if (!est_filein.IsNull()) - mempool.ReadFeeEstimates(est_filein); + ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; // ********************************************************* Step 8: load wallet @@ -1603,9 +1613,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); std::vector<fs::path> vImportFiles; - if (mapMultiArgs.count("-loadblock")) + if (gArgs.IsArgSet("-loadblock")) { - BOOST_FOREACH(const std::string& strFile, mapMultiArgs.at("-loadblock")) + BOOST_FOREACH(const std::string& strFile, gArgs.GetArgs("-loadblock")) vImportFiles.push_back(strFile); } diff --git a/src/key.cpp b/src/key.cpp index b4f0dc8202..5a75647f1a 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -131,14 +131,6 @@ void CKey::MakeNewKey(bool fCompressedIn) { fCompressed = fCompressedIn; } -bool CKey::SetPrivKey(const CPrivKey &privkey, bool fCompressedIn) { - if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), &privkey[0], privkey.size())) - return false; - fCompressed = fCompressedIn; - fValid = true; - return true; -} - CPrivKey CKey::GetPrivKey() const { assert(fValid); CPrivKey privkey; @@ -94,9 +94,6 @@ public: //! Check whether the public key corresponding to this private key is (to be) compressed. bool IsCompressed() const { return fCompressed; } - //! Initialize from a CPrivKey (serialized OpenSSL private key data). - bool SetPrivKey(const CPrivKey& vchPrivKey, bool fCompressed); - //! Generate a new private key using a cryptographic PRNG. void MakeNewKey(bool fCompressed); diff --git a/src/memusage.h b/src/memusage.h index 81e8702954..b69acafffd 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -12,6 +12,8 @@ #include <map> #include <set> #include <vector> +#include <unordered_map> +#include <unordered_set> #include <boost/foreach.hpp> #include <boost/unordered_set.hpp> @@ -149,7 +151,7 @@ static inline size_t DynamicUsage(const std::shared_ptr<X>& p) // Boost data structures template<typename X> -struct boost_unordered_node : private X +struct unordered_node : private X { private: void* ptr; @@ -158,13 +160,25 @@ private: template<typename X, typename Y> static inline size_t DynamicUsage(const boost::unordered_set<X, Y>& s) { - return MallocUsage(sizeof(boost_unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); } template<typename X, typename Y, typename Z> static inline size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& m) { - return MallocUsage(sizeof(boost_unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::unordered_set<X, Y>& s) +{ + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const std::unordered_map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } } diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index e3f3e4621a..78d7cd6001 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -65,7 +65,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve } else { // calculate left hash uint256 left = CalcHash(height-1, pos*2, vTxid), right; - // calculate right hash if not beyond the end of the array - copy left hash otherwise1 + // calculate right hash if not beyond the end of the array - copy left hash otherwise if (pos*2+1 < CalcTreeWidth(height-1)) right = CalcHash(height-1, pos*2+1, vTxid); else diff --git a/src/merkleblock.h b/src/merkleblock.h index 73cbf670ee..de4c5c8d29 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -23,7 +23,7 @@ * storing a bit for each traversed node, signifying whether the node is the * parent of at least one matched leaf txid (or a matched txid itself). In * case we are at the leaf level, or this bit is 0, its merkle node hash is - * stored, and its children are not explorer further. Otherwise, no hash is + * stored, and its children are not explored further. Otherwise, no hash is * stored, but we recurse into both (or the only) child branch. During * decoding, the same depth-first traversal is performed, consuming bits and * hashes as they written during encoding. diff --git a/src/miner.cpp b/src/miner.cpp index 8f17c575cb..28b6f23d56 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -16,6 +16,7 @@ #include "hash.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "pow.h" #include "primitives/transaction.h" @@ -318,9 +319,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) { assert (it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) - return true; - return false; + return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); } void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries) diff --git a/src/net.cpp b/src/net.cpp index cf94faf854..ded6f1099a 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -418,10 +418,10 @@ void CConnman::DumpBanlist() CBanDB bandb; banmap_t banmap; - SetBannedSetDirty(false); GetBanned(banmap); - if (!bandb.Write(banmap)) - SetBannedSetDirty(true); + if (bandb.Write(banmap)) { + SetBannedSetDirty(false); + } LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); @@ -541,6 +541,8 @@ bool CConnman::Unban(const CSubNet &subNet) { void CConnman::GetBanned(banmap_t &banMap) { LOCK(cs_setBanned); + // Sweep the banlist so expired bans are not returned + SweepBanned(); banMap = setBanned; //create a thread safe copy } @@ -708,7 +710,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete handled = msg.readData(pch, nBytes); if (handled < 0) - return false; + return false; if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); @@ -786,7 +788,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) // reject messages larger than MAX_SIZE if (hdr.nMessageSize > MAX_SIZE) - return -1; + return -1; // switch state to reading message data in_data = true; @@ -946,7 +948,7 @@ bool CConnman::AttemptToEvictConnection() continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, (node->nServices & nRelevantServices) == nRelevantServices, node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; @@ -1069,12 +1071,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // According to the internet TCP_NODELAY is not carried into accepted sockets // on all platforms. Set it again here just to be sure. - int set = 1; -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); if (IsBanned(addr) && !whitelisted) { @@ -1299,59 +1296,55 @@ void CConnman::ThreadSocketHandler() } if (recvSet || errorSet) { + // typical socket buffer is 8K-64K + char pchBuf[0x10000]; + int nBytes = 0; { - { - // typical socket buffer is 8K-64K - char pchBuf[0x10000]; - int nBytes = 0; - { - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) - continue; - nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); - } - if (nBytes > 0) - { - bool notify = false; - if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) - pnode->CloseSocketDisconnect(); - RecordBytesRecv(nBytes); - if (notify) { - size_t nSizeAdded = 0; - auto it(pnode->vRecvMsg.begin()); - for (; it != pnode->vRecvMsg.end(); ++it) { - if (!it->complete()) - break; - nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; - } - { - LOCK(pnode->cs_vProcessMsg); - pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); - pnode->nProcessQueueSize += nSizeAdded; - pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; - } - WakeMessageHandler(); - } + LOCK(pnode->cs_hSocket); + if (pnode->hSocket == INVALID_SOCKET) + continue; + nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); + } + if (nBytes > 0) + { + bool notify = false; + if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) + pnode->CloseSocketDisconnect(); + RecordBytesRecv(nBytes); + if (notify) { + size_t nSizeAdded = 0; + auto it(pnode->vRecvMsg.begin()); + for (; it != pnode->vRecvMsg.end(); ++it) { + if (!it->complete()) + break; + nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; } - else if (nBytes == 0) { - // socket closed gracefully - if (!pnode->fDisconnect) { - LogPrint(BCLog::NET, "socket closed\n"); - } - pnode->CloseSocketDisconnect(); - } - else if (nBytes < 0) - { - // error - int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) - { - if (!pnode->fDisconnect) - LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); - pnode->CloseSocketDisconnect(); - } + LOCK(pnode->cs_vProcessMsg); + pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); + pnode->nProcessQueueSize += nSizeAdded; + pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; } + WakeMessageHandler(); + } + } + else if (nBytes == 0) + { + // socket closed gracefully + if (!pnode->fDisconnect) { + LogPrint(BCLog::NET, "socket closed\n"); + } + pnode->CloseSocketDisconnect(); + } + else if (nBytes < 0) + { + // error + int nErr = WSAGetLastError(); + if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) + { + if (!pnode->fDisconnect) + LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); + pnode->CloseSocketDisconnect(); } } } @@ -1376,7 +1369,7 @@ void CConnman::ThreadSocketHandler() { if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) { - LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->id); + LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->GetId()); pnode->fDisconnect = true; } else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) @@ -1396,7 +1389,7 @@ void CConnman::ThreadSocketHandler() } else if (!pnode->fSuccessfullyConnected) { - LogPrintf("version handshake timeout from %d\n", pnode->id); + LogPrintf("version handshake timeout from %d\n", pnode->GetId()); pnode->fDisconnect = true; } } @@ -1585,6 +1578,9 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); BOOST_FOREACH(const CDNSSeedData &seed, vSeeds) { + if (interruptNet) { + return; + } if (HaveNameProxy()) { AddOneShot(seed.host); } else { @@ -1602,6 +1598,9 @@ void CConnman::ThreadDNSAddressSeed() found++; } } + if (interruptNet) { + return; + } // TODO: The seed name resolve may fail, yielding an IP of [::], which results in // addrman assigning the same source to results from different seeds. // This should switch to a hard-coded stable dummy IP for each seed name, so that the @@ -1666,12 +1665,12 @@ void CConnman::ProcessOneShot() void CConnman::ThreadOpenConnections() { // Connect to specific addresses - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) + if (gArgs.IsArgSet("-connect") && gArgs.GetArgs("-connect").size() > 0) { for (int64_t nLoop = 0;; nLoop++) { ProcessOneShot(); - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-connect")) + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-connect")) { CAddress addr(CService(), NODE_NONE); OpenNetworkConnection(addr, false, NULL, strAddr.c_str()); @@ -1873,8 +1872,8 @@ void CConnman::ThreadOpenAddedConnections() { { LOCK(cs_vAddedNodes); - if (mapMultiArgs.count("-addnode")) - vAddedNodes = mapMultiArgs.at("-addnode"); + if (gArgs.IsArgSet("-addnode")) + vAddedNodes = gArgs.GetArgs("-addnode"); } while (true) @@ -2285,7 +2284,7 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c threadOpenAddedConnections = std::thread(&TraceThread<std::function<void()> >, "addcon", std::function<void()>(std::bind(&CConnman::ThreadOpenAddedConnections, this))); // Initiate outbound connections unless connect=0 - if (!mapMultiArgs.count("-connect") || mapMultiArgs.at("-connect").size() != 1 || mapMultiArgs.at("-connect")[0] != "0") + if (!gArgs.IsArgSet("-connect") || gArgs.GetArgs("-connect").size() != 1 || gArgs.GetArgs("-connect")[0] != "0") threadOpenConnections = std::thread(&TraceThread<std::function<void()> >, "opencon", std::function<void()>(std::bind(&CConnman::ThreadOpenConnections, this))); // Process messages @@ -2483,7 +2482,7 @@ bool CConnman::DisconnectNode(NodeId id) { LOCK(cs_vNodes); for(CNode* pnode : vNodes) { - if (id == pnode->id) { + if (id == pnode->GetId()) { pnode->fDisconnect = true; return true; } @@ -2621,10 +2620,10 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), fInbound(fInboundIn), - id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001), + id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn), @@ -2740,7 +2739,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->GetId()); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -2778,7 +2777,7 @@ bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) CNode* found = nullptr; LOCK(cs_vNodes); for (auto&& pnode : vNodes) { - if(pnode->id == id) { + if(pnode->GetId() == id) { found = pnode; break; } @@ -11,10 +11,10 @@ #include "amount.h" #include "bloom.h" #include "compat.h" -#include "fs.h" #include "hash.h" #include "limitedmap.h" #include "netaddress.h" +#include "policy/feerate.h" #include "protocol.h" #include "random.h" #include "streams.h" @@ -36,7 +36,6 @@ #include <boost/foreach.hpp> #include <boost/signals2/signal.hpp> -class CAddrMan; class CScheduler; class CNode; @@ -92,7 +91,7 @@ static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban -typedef int NodeId; +typedef int64_t NodeId; struct AddedNodeInfo { @@ -613,7 +612,6 @@ public: CCriticalSection cs_filter; CBloomFilter* pfilter; std::atomic<int> nRefCount; - const NodeId id; const uint64_t nKeyedNetGroup; std::atomic_bool fPauseRecv; @@ -684,6 +682,7 @@ public: private: CNode(const CNode&); void operator=(const CNode&); + const NodeId id; const uint64_t nLocalHostNonce; @@ -701,15 +700,15 @@ private: public: NodeId GetId() const { - return id; + return id; } uint64_t GetLocalNonce() const { - return nLocalHostNonce; + return nLocalHostNonce; } int GetMyStartingHeight() const { - return nMyStartingHeight; + return nMyStartingHeight; } int GetRefCount() @@ -760,7 +759,7 @@ public: // after addresses were pushed. if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { - vAddrToSend[insecure_rand.rand32() % vAddrToSend.size()] = _addr; + vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { vAddrToSend.push_back(_addr); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 17653f542d..aca88f2660 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -303,6 +303,7 @@ void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); } + LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } // Requires cs_main. @@ -333,7 +334,7 @@ bool MarkBlockAsReceived(const uint256& hash) { // Requires cs_main. // returns false, still setting pit, if the block was already in flight from the same peer // pit will only be valid as long as the same cs_main lock is being held -bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); @@ -744,21 +745,23 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanI recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); } -void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock) { - if (nPosInBlock == CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK) - return; - +void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { LOCK(cs_main); std::vector<uint256> vOrphanErase; - // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = *(*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); + + for (const CTransactionRef& ptx : pblock->vtx) { + const CTransaction& tx = *ptx; + + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = *(*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } } } @@ -772,10 +775,12 @@ void PeerLogicValidation::SyncTransaction(const CTransaction& tx, const CBlockIn } } +// All of the following cache a recent block, and are protected by cs_most_recent_block static CCriticalSection cs_most_recent_block; static std::shared_ptr<const CBlock> most_recent_block; static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block; static uint256 most_recent_block_hash; +static bool fWitnessesPresentInMostRecentCompactBlock; void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); @@ -796,6 +801,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: most_recent_block_hash = hashBlock; most_recent_block = pblock; most_recent_compact_block = pcmpctblock; + fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { @@ -810,7 +816,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", - hashBlock.ToString(), pnode->id); + hashBlock.ToString(), pnode->GetId()); connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; } @@ -856,8 +862,8 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta int nDoS = 0; if (state.IsInvalid(nDoS)) { - if (it != mapBlockSource.end() && State(it->second.first)) { - assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + // Don't send reject message with code 0 or an internal reject code. + if (it != mapBlockSource.end() && State(it->second.first) && state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) { CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), hash}; State(it->second.first)->rejects.push_back(reject); if (nDoS > 0 && it->second.second) @@ -945,7 +951,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { if (pnode->nVersion >= CADDR_TIME_VERSION) { - uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); @@ -988,6 +994,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + std::shared_ptr<const CBlock> a_recent_block; + std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; + bool fWitnessesPresentInARecentCompactBlock; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + a_recent_compact_block = most_recent_compact_block; + fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; + } if (mi != mapBlockIndex.end()) { if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && @@ -997,11 +1012,6 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // before ActivateBestChain but after AcceptBlock). // In this case, we need to run ActivateBestChain prior to checking the relay // conditions below. - std::shared_ptr<const CBlock> a_recent_block; - { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - } CValidationState dummy; ActivateBestChain(dummy, Params(), a_recent_block); } @@ -1035,14 +1045,20 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // it's available before trying to send. if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { - // Send block from disk - CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) - assert(!"cannot load block from disk"); + std::shared_ptr<const CBlock> pblock; + if (a_recent_block && a_recent_block->GetHash() == (*mi).second->GetBlockHash()) { + pblock = a_recent_block; + } else { + // Send block from disk + std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); + if (!ReadBlockFromDisk(*pblockRead, (*mi).second, consensusParams)) + assert(!"cannot load block from disk"); + pblock = pblockRead; + } if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1051,7 +1067,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam LOCK(pfrom->cs_filter); if (pfrom->pfilter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(block, *pfrom->pfilter); + merkleBlock = CMerkleBlock(*pblock, *pfrom->pfilter); } } if (sendMerkleBlock) { @@ -1064,7 +1080,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1078,10 +1094,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { - CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - } else - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); + if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + } else { + CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + } + } else { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + } } // Trigger the peer node to send a getblocks request for the next batch of inventory @@ -1142,7 +1163,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } -uint32_t GetFetchFlags(CNode* pfrom, const CBlockIndex* pprev, const Consensus::Params& chainparams) { +uint32_t GetFetchFlags(CNode* pfrom) { uint32_t nFetchFlags = 0; if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; @@ -1156,7 +1177,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac if (req.indexes[i] >= block.vtx.size()) { LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->GetId()); return; } resp.txn[i] = block.vtx[req.indexes[i]]; @@ -1169,7 +1190,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic<bool>& interruptMsgProc) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (IsArgSet("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1248,7 +1269,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nServicesExpected & ~nServices) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); pfrom->fDisconnect = true; @@ -1258,7 +1279,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, nVersion); + LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; @@ -1360,7 +1381,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom->nVersion, - pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, + pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -1530,9 +1551,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); - - std::vector<CInv> vToFetch; + uint32_t nFetchFlags = GetFetchFlags(pfrom); for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { @@ -1542,7 +1561,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; bool fAlreadyHave = AlreadyHave(inv); - LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->GetId()); if (inv.type == MSG_TX) { inv.type |= nFetchFlags; @@ -1557,14 +1576,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); } } else { pfrom->AddInventoryKnown(inv); if (fBlocksOnly) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->GetId()); } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { pfrom->AskFor(inv); } @@ -1573,9 +1592,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Track requests for our stuff GetMainSignals().Inventory(inv.hash); } - - if (!vToFetch.empty()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); } @@ -1590,10 +1606,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return error("message getdata size() = %u", vInv.size()); } - LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); + LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->GetId()); if (vInv.size() > 0) { - LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->GetId()); } pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); @@ -1633,7 +1649,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); + LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) @@ -1683,7 +1699,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); return true; } @@ -1695,7 +1711,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // might maliciously send lots of getblocktxn requests to trigger // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. - LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->GetId(), MAX_BLOCKTXN_DEPTH); CInv inv; inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; @@ -1720,7 +1736,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->GetId()); return true; } @@ -1745,7 +1761,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; int nLimit = MAX_HEADERS_RESULTS; - LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -1775,7 +1791,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { - LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->GetId()); return true; } @@ -1808,7 +1824,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->nLastTXTime = GetTime(); LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", - pfrom->id, + pfrom->GetId(), tx.GetHash().ToString(), mempool.size(), mempool.DynamicMemoryUsage() / 1000); @@ -1883,7 +1899,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } if (!fRejectedParents) { - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); BOOST_FOREACH(const CTxIn& txin, tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); pfrom->AddInventoryKnown(_inv); @@ -1928,10 +1944,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // case. int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); RelayTransaction(tx, connman); } else { - LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } } } @@ -1943,9 +1959,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (state.IsInvalid(nDoS)) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), - pfrom->id, + pfrom->GetId(), FormatStateMessage(state)); - if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P + if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash)); if (nDoS > 0) { @@ -1980,7 +1996,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); return true; } } @@ -2020,7 +2036,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; @@ -2044,7 +2060,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = NULL; - if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); else { @@ -2059,12 +2075,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2107,7 +2123,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { @@ -2164,7 +2180,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || it->second.first != pfrom->GetId()) { - LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->GetId()); return true; } @@ -2173,12 +2189,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received @@ -2261,7 +2277,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), pindexBestHeader->nHeight, - pfrom->id, nodestate->nUnconnectingHeaders); + pfrom->GetId(), nodestate->nUnconnectingHeaders); // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2299,7 +2315,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); CNodeState *nodestate = State(pfrom->GetId()); if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); } nodestate->nUnconnectingHeaders = 0; @@ -2310,7 +2326,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } @@ -2346,11 +2362,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Can't download any more from this peer break; } - uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->id); + pindex->GetBlockHash().ToString(), pfrom->GetId()); } if (vGetData.size() > 1) { LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", @@ -2373,7 +2389,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; - LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. @@ -2405,14 +2421,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. if (!pfrom->fInbound) { - LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->GetId()); return true; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (pfrom->fSentAddr) { - LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->GetId()); return true; } pfrom->fSentAddr = true; @@ -2513,7 +2529,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!(sProblem.empty())) { LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", - pfrom->id, + pfrom->GetId(), sProblem, pfrom->nPingNonceSent, nonce, @@ -2590,7 +2606,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(pfrom->cs_feeFilter); pfrom->minFeeFilter = newFeeFilter; } - LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->GetId()); } } @@ -2601,7 +2617,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); } @@ -2661,100 +2677,100 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + // Don't bother if send buffer is too full to respond anyway + if (pfrom->fPauseSend) + return false; - std::list<CNetMessage> msgs; - { - LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; - // Just take one message - msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); - pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); - fMoreWork = !pfrom->vProcessMsg.empty(); - } - CNetMessage& msg(msgs.front()); - - msg.SetVersion(pfrom->GetRecvVersion()); - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); - pfrom->fDisconnect = true; + std::list<CNetMessage> msgs; + { + LOCK(pfrom->cs_vProcessMsg); + if (pfrom->vProcessMsg.empty()) return false; - } + // Just take one message + msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); + pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; + pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); + fMoreWork = !pfrom->vProcessMsg.empty(); + } + CNetMessage& msg(msgs.front()); + + msg.SetVersion(pfrom->GetRecvVersion()); + // Scan for message start + if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + pfrom->fDisconnect = true; + return false; + } - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) - { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); - return fMoreWork; - } - std::string strCommand = hdr.GetCommand(); + // Read header + CMessageHeader& hdr = msg.hdr; + if (!hdr.IsValid(chainparams.MessageStart())) + { + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + return fMoreWork; + } + std::string strCommand = hdr.GetCommand(); - // Message size - unsigned int nMessageSize = hdr.nMessageSize; + // Message size + unsigned int nMessageSize = hdr.nMessageSize; - // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + // Checksum + CDataStream& vRecv = msg.vRecv; + const uint256& hash = msg.GetMessageHash(); + if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + { + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + SanitizeString(strCommand), nMessageSize, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + return fMoreWork; + } + + // Process message + bool fRet = false; + try + { + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); + if (interruptMsgProc) + return false; + if (!pfrom->vRecvGetData.empty()) + fMoreWork = true; + } + catch (const std::ios_base::failure& e) + { + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + if (strstr(e.what(), "end of data")) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); - return fMoreWork; + // Allow exceptions from under-length message on vRecv + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - - // Process message - bool fRet = false; - try + else if (strstr(e.what(), "size too large")) { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); - if (interruptMsgProc) - return false; - if (!pfrom->vRecvGetData.empty()) - fMoreWork = true; + // Allow exceptions from over-long size + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::ios_base::failure& e) + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); - if (strstr(e.what(), "end of data")) - { - // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "size too large")) - { - // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "non-canonical ReadCompactSize()")) - { - // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else - { - PrintExceptionContinue(&e, "ProcessMessages()"); - } + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::exception& e) { + else + { PrintExceptionContinue(&e, "ProcessMessages()"); - } catch (...) { - PrintExceptionContinue(NULL, "ProcessMessages()"); } + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (...) { + PrintExceptionContinue(NULL, "ProcessMessages()"); + } - if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); - } + if (!fRet) { + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + } - LOCK(cs_main); - SendRejectsAndCheckIfBanned(pfrom, connman); + LOCK(cs_main); + SendRejectsAndCheckIfBanned(pfrom, connman); return fMoreWork; } @@ -2879,7 +2895,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -2909,7 +2925,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = NULL; // last header queued for delivery - ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date + ProcessBlockAvailability(pto->GetId()); // ensure pindexBestKnownBlock is up-to-date if (!fRevertToInv) { bool fFoundStartingHeader = false; @@ -2964,7 +2980,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // We only send up to 1 block as header-and-ids, as otherwise // probably means we're doing an initial-ish-sync or they're slow LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + vHeaders.front().GetHash().ToString(), pto->GetId()); int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; @@ -2972,7 +2988,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - if (state.fWantsCmpctWitness) + if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); else { CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); @@ -2994,10 +3010,10 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr LogPrint(BCLog::NET, "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, vHeaders.size(), vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); + vHeaders.back().GetHash().ToString(), pto->GetId()); } else { LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + vHeaders.front().GetHash().ToString(), pto->GetId()); } connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; @@ -3026,7 +3042,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, - pto->id, hashToAnnounce.ToString()); + pto->GetId(), hashToAnnounce.ToString()); } } } @@ -3173,7 +3189,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); pto->fDisconnect = true; return true; } @@ -3186,7 +3202,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->GetId()); pto->fDisconnect = true; return true; } @@ -3201,11 +3217,11 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); BOOST_FOREACH(const CBlockIndex *pindex, vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + uint32_t nFetchFlags = GetFetchFlags(pto); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); + pindex->nHeight, pto->GetId()); } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { @@ -3223,7 +3239,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) { - LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->id); + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); vGetData.push_back(inv); if (vGetData.size() >= 1000) { diff --git a/src/net_processing.h b/src/net_processing.h index 9e3f1b7156..f460595bc1 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -30,10 +30,10 @@ private: public: PeerLogicValidation(CConnman* connmanIn); - virtual void SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int nPosInBlock); - virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); - virtual void BlockChecked(const CBlock& block, const CValidationState& state); - virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock); + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; + void BlockChecked(const CBlock& block, const CValidationState& state) override; + void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; }; struct CNodeStateStats { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index ab07270f3f..34a7029862 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -179,12 +179,6 @@ bool CNetAddr::IsLocal() const return false; } -bool CNetAddr::IsMulticast() const -{ - return (IsIPv4() && (GetByte(3) & 0xF0) == 0xE0) - || (GetByte(15) == 0xFF); -} - bool CNetAddr::IsValid() const { // Cleanup 3-byte shifted addresses caused by garbage in size field diff --git a/src/netaddress.h b/src/netaddress.h index a85c2b7452..fbc4d1a65f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -65,7 +65,6 @@ class CNetAddr bool IsLocal() const; bool IsRoutable() const; bool IsValid() const; - bool IsMulticast() const; enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; diff --git a/src/netbase.cpp b/src/netbase.cpp index bdc725359a..2584f571ee 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -428,18 +428,14 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe if (hSocket == INVALID_SOCKET) return false; - int set = 1; #ifdef SO_NOSIGPIPE + int set = 1; // Different way of disabling SIGPIPE on BSD setsockopt(hSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)); #endif //Disable Nagle's algorithm -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) @@ -728,6 +724,13 @@ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking) return true; } +bool SetSocketNoDelay(SOCKET& hSocket) +{ + int set = 1; + int rc = setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); + return rc == 0; +} + void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index dd33b6e47e..c9d108aadd 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -59,6 +59,8 @@ std::string NetworkErrorString(int err); bool CloseSocket(SOCKET& hSocket); /** Disable or enable blocking-mode for a socket */ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking); +/** Set the TCP_NODELAY flag on a socket */ +bool SetSocketNoDelay(SOCKET& hSocket); /** * Convert milliseconds to a struct timeval for e.g. select. */ diff --git a/src/amount.cpp b/src/policy/feerate.cpp index a5f6bc3cd9..a089c02284 100644 --- a/src/amount.cpp +++ b/src/policy/feerate.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" +#include "feerate.h" #include "tinyformat.h" diff --git a/src/policy/feerate.h b/src/policy/feerate.h new file mode 100644 index 0000000000..e82268b095 --- /dev/null +++ b/src/policy/feerate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_FEERATE_H +#define BITCOIN_POLICY_FEERATE_H + +#include "amount.h" +#include "serialize.h" + +#include <string> + +extern const std::string CURRENCY_UNIT; + +/** + * Fee rate in satoshis per kilobyte: CAmount / kB + */ +class CFeeRate +{ +private: + CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes +public: + /** Fee rate of 0 satoshis per kB */ + CFeeRate() : nSatoshisPerK(0) { } + explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } + /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ + CFeeRate(const CAmount& nFeePaid, size_t nBytes); + CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } + /** + * Return the fee in satoshis for the given size in bytes. + */ + CAmount GetFee(size_t nBytes) const; + /** + * Return the fee in satoshis for a size of 1000 bytes + */ + CAmount GetFeePerK() const { return GetFee(1000); } + friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } + friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } + friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } + friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } + friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } + std::string ToString() const; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(nSatoshisPerK); + } +}; + +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 38e07dc345..bad3de4b44 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -7,46 +7,163 @@ #include "policy/policy.h" #include "amount.h" +#include "clientversion.h" #include "primitives/transaction.h" #include "random.h" #include "streams.h" #include "txmempool.h" #include "util.h" -void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay) +static constexpr double INF_FEERATE = 1e99; + +/** + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into + const std::vector<double>& buckets; // The upper-bound of the range for the bucket (inclusive) + const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> txCtAvg; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector<std::vector<double>> confAvg; // confAvg[Y][X] + + // Track moving avg of txs which have been evicted from the mempool + // after failing to be confirmed within Y blocks + std::vector<std::vector<double>> failAvg; // failAvg[Y][X] + + // Sum the total feerate of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> avg; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg feerate per bucket + + double decay; + + // Resolution (# of blocks) with which confirmations are tracked + unsigned int scale; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after GetMaxConfirms for each bucket + std::vector<int> oldUnconfTxs; + + void resizeInMemoryCounters(size_t newbuckets); + +public: + /** + * Create new TxConfirmStats. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundaries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + */ + TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double decay, unsigned int scale); + + /** Roll the circular buffer for unconfirmed txs*/ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val the feerate of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex, bool inBlock); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight, + EstimationResult *result = nullptr) const; + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout) const; + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); +}; + + +TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, + const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double _decay, unsigned int _scale) + : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; - for (unsigned int i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { + scale = _scale; + confAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); + } + failAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { + failAvg[i].resize(buckets.size()); } - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); + + resizeInMemoryCounters(buckets.size()); +} + +void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) { + // newbuckets must be passed in because the buckets referred to during Read have not been updated yet. + unconfTxs.resize(GetMaxConfirms()); + for (unsigned int i = 0; i < unconfTxs.size(); i++) { + unconfTxs[i].resize(newbuckets); + } + oldUnconfTxs.resize(newbuckets); } -// Zero out the data for the current block +// Roll the unconfirmed txs circular buffer void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (unsigned int j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; - for (unsigned int i = 0; i < curBlockConf.size(); i++) - curBlockConf[i][j] = 0; - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; } } @@ -56,33 +173,38 @@ void TxConfirmStats::Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + scale - 1)/scale; unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; + for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) { + confAvg[i - 1][bucketindex]++; } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; + txCtAvg[bucketindex]++; + avg[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + confAvg[i][j] = confAvg[i][j] * decay; + for (unsigned int i = 0; i < failAvg.size(); i++) + failAvg[i][j] = failAvg[i][j] * decay; + avg[j] = avg[j] * decay; + txCtAvg[j] = txCtAvg[j] * decay; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) + unsigned int nBlockHeight, EstimationResult *result) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + scale - 1)/scale; int maxbucketindex = buckets.size() - 1; @@ -105,12 +227,21 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket; + EstimatorBucket failBucket; // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; + nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; + failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; @@ -119,24 +250,41 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { + if (passing == true) { + // First time we hit a failure record the failed bucket + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters else { + failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; + passing = true; + passBucket.withinTarget = nConf; nConf = 0; + passBucket.totalConfirmed = totalNum; totalNum = 0; + passBucket.inMempool = extraNum; + passBucket.leftMempool = failNum; + failNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + newBucketRange = true; } } } @@ -148,8 +296,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int minBucket = std::min(bestNearBucket, bestFarBucket); + unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket); for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } @@ -163,83 +311,109 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, break; } } + + passBucket.start = minBucket ? buckets[minBucket-1] : 0; + passBucket.end = buckets[maxBucket]; } - LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], - 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) { + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + } + LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d %s%.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + confTarget, requireGreater ? ">" : "<", 100.0 * successBreakPoint, decay, + median, passBucket.start, passBucket.end, + 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool), + passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool, + failBucket.start, failBucket.end, + 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool), + failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool); + + + if (result) { + result->pass = passBucket; + result->fail = failBucket; + result->decay = decay; + result->scale = scale; + } return median; } -void TxConfirmStats::Write(CAutoFile& fileout) +void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; - fileout << buckets; + fileout << scale; fileout << avg; fileout << txCtAvg; fileout << confAvg; + fileout << failAvg; } -void TxConfirmStats::Read(CAutoFile& filein) +void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) { - // Read data file into temporary variables and do some very basic sanity checking - std::vector<double> fileBuckets; - std::vector<double> fileAvg; - std::vector<std::vector<double> > fileConfAvg; - std::vector<double> fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - filein >> fileAvg; - if (fileAvg.size() != numBuckets) + // Read data file and do some very basic sanity checking + // buckets and bucketMap are not updated yet, so don't access them + // If there is a read failure, we'll just discard this entire object anyway + size_t maxConfirms, maxPeriods; + + // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor + if (nFileVersion >= 149900) { + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + } + + filein >> avg; + if (avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) + } + filein >> txCtAvg; + if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week - throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); + filein >> confAvg; + maxPeriods = confAvg.size(); + maxConfirms = scale * maxPeriods; - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (confAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); + if (nFileVersion >= 149900) { + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + } else { + failAvg.resize(confAvg.size()); + for (unsigned int i = 0; i < failAvg.size(); i++) { + failAvg[i].resize(numBuckets); + } } - oldUnconfTxs.resize(buckets.size()); - for (unsigned int i = 0; i < buckets.size(); i++) - bucketMap[buckets[i]] = i; + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + resizeInMemoryCounters(numBuckets); LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); @@ -253,7 +427,7 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) return bucketindex; } -void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; @@ -281,6 +455,12 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe blockIndex, bucketindex); } } + if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + unsigned int periodsAgo = blocksAgo / scale; + for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { + failAvg[i][bucketindex]++; + } + } } // This function is called from CTxMemPool::removeUnchecked to ensure @@ -288,11 +468,14 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // tracked. Txs that were part of a block have already been removed in // processBlockTx to ensure they are never double tracked, but it is // of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) +bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { + LOCK(cs_feeEstimator); std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); mapMemPoolTxs.erase(hash); return true; } else { @@ -301,20 +484,33 @@ bool CBlockPolicyEstimator::removeTx(uint256 hash) } CBlockPolicyEstimator::CBlockPolicyEstimator() - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) + : nBestSeenHeight(0), firstRecordedHeight(0), historicalFirst(0), historicalBest(0), trackedTxs(0), untrackedTxs(0) { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); - minTrackedFee = CFeeRate(MIN_BUCKET_FEERATE); - std::vector<double> vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); + size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { + buckets.push_back(bucketBoundary); + bucketMap[bucketBoundary] = bucketIndex; } - vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + buckets.push_back(INF_FEERATE); + bucketMap[INF_FEERATE] = bucketIndex; + assert(bucketMap.size() == buckets.size()); + + feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); + shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); + longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); +} + +CBlockPolicyEstimator::~CBlockPolicyEstimator() +{ + delete feeStats; + delete shortStats; + delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) { + LOCK(cs_feeEstimator); unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { @@ -343,12 +539,17 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = bucketIndex; + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex2); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex3); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash())) { + if (!removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -367,13 +568,16 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries) { + LOCK(cs_feeEstimator); if (nBlockHeight <= nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random // they don't affect the estimate. @@ -388,34 +592,77 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, // of unconfirmed txs to remove from tracking. nBestSeenHeight = nBlockHeight; - // Clear the current block state and update unconfirmed circular buffer - feeStats.ClearCurrent(nBlockHeight); + // Update unconfirmed circular buffer + feeStats->ClearCurrent(nBlockHeight); + shortStats->ClearCurrent(nBlockHeight); + longStats->ClearCurrent(nBlockHeight); + + // Decay all exponential averages + feeStats->UpdateMovingAverages(); + shortStats->UpdateMovingAverages(); + longStats->UpdateMovingAverages(); unsigned int countedTxs = 0; - // Repopulate the current block states + // Update averages with data points from current block for (unsigned int i = 0; i < entries.size(); i++) { if (processBlockTx(nBlockHeight, entries[i])) countedTxs++; } - // Update all exponential averages with the current block state - feeStats.UpdateMovingAverages(); + if (firstRecordedHeight == 0 && countedTxs > 0) { + firstRecordedHeight = nBestSeenHeight; + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight); + } + - LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", + countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; untrackedTxs = 0; } -CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { - // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + if (confTarget <= 1) + return CFeeRate(0); + + return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE); +} + +CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const +{ + TxConfirmStats* stats; + double sufficientTxs = SUFFICIENT_FEETXS; + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + stats = shortStats; + sufficientTxs = SUFFICIENT_TXS_SHORT; + break; + } + case FeeEstimateHorizon::MED_HALFLIFE: { + stats = feeStats; + break; + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + stats = longStats; + break; + } + default: { return CFeeRate(0); + } + } - double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + LOCK(cs_feeEstimator); + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms()) + return CFeeRate(0); + if (successThreshold > 1) + return CFeeRate(0); + + double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, nBestSeenHeight, result); if (median < 0) return CFeeRate(0); @@ -423,25 +670,148 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) +unsigned int CBlockPolicyEstimator::BlockSpan() const +{ + if (firstRecordedHeight == 0) return 0; + assert(nBestSeenHeight >= firstRecordedHeight); + + return nBestSeenHeight - firstRecordedHeight; +} + +unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const +{ + if (historicalFirst == 0) return 0; + assert(historicalBest >= historicalFirst); + + if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0; + + return historicalBest - historicalFirst; +} + +unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const +{ + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2); +} + +/** Return a fee estimate at the required successThreshold from the shortest + * time horizon which tracks confirmations up to the desired target. If + * checkShorterHorizon is requested, also allow short time horizon estimates + * for a lower target to reduce the given answer */ +double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const +{ + double estimate = -1; + if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) { + // Find estimate from shortest time horizon possible + if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon + estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + } + else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon + estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + else { // long horizon + estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + if (checkShorterHorizon) { + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > feeStats->GetMaxConfirms()) { + double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + estimate = medMax; + } + if (confTarget > shortStats->GetMaxConfirms()) { + double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + estimate = shortMax; + } + } + } + return estimate; +} + +/** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + * at 2 * target for any longer time horizons. + */ +double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget) const +{ + double estimate = -1; + if (doubleTarget <= shortStats->GetMaxConfirms()) { + estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + } + if (doubleTarget <= feeStats->GetMaxConfirms()) { + double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + if (longEstimate > estimate) { + estimate = longEstimate; + } + } + return estimate; +} + +/** estimateSmartFee returns the max of the feerates calculated with a 60% + * threshold required at target / 2, an 85% threshold required at target and a + * 95% threshold required at 2 * target. Each calculation is performed at the + * shortest time horizon which tracks the required target. Conservative + * estimates, however, required the 95% threshold at 2 * target be met for any + * longer time horizons also. + */ +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) - return CFeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; double median = -1; - while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { - median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } + { + LOCK(cs_feeEstimator); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) + return CFeeRate(0); + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + unsigned int maxUsableEstimate = MaxUsableEstimate(); + if (maxUsableEstimate <= 1) + return CFeeRate(0); + + if ((unsigned int)confTarget > maxUsableEstimate) { + confTarget = maxUsableEstimate; + } + + assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + /** true is passed to estimateCombined fee for target/2 and target so + * that we check the max confirms for shorter time horizons as well. + * This is necessary to preserve monotonically increasing estimates. + * For non-conservative estimates we do the same thing for 2*target, but + * for conservative estimates we want to skip these shorter horizons + * checks for 2*target becuase we are taking the max over all time + * horizons so we already have monotonically increasing estimates and + * the purpose of conservative estimates is not to let short term + * fluctuations lower our estimates by too much. + */ + double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true); + double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true); + double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative); + median = halfEst; + if (actualEst > median) { + median = actualEst; + } + if (doubleEst > median) { + median = doubleEst; + } + + if (conservative || median == -1) { + double consEst = estimateConservativeFee(2 * confTarget); + if (consEst > median) { + median = consEst; + } + } + } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; + *answerFoundAtTarget = confTarget; // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); @@ -454,26 +824,134 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } -void CBlockPolicyEstimator::Write(CAutoFile& fileout) + +bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { - fileout << nBestSeenHeight; - feeStats.Write(fileout); + try { + LOCK(cs_feeEstimator); + fileout << 149900; // version required to read: 0.14.99 or later + fileout << CLIENT_VERSION; // version that wrote the file + fileout << nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()/2) { + fileout << firstRecordedHeight << nBestSeenHeight; + } + else { + fileout << historicalFirst << historicalBest; + } + fileout << buckets; + feeStats->Write(fileout); + shortStats->Write(fileout); + longStats->Write(fileout); + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); + return false; + } + return true; } -void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) +bool CBlockPolicyEstimator::Read(CAutoFile& filein) { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - feeStats.Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + try { + LOCK(cs_feeEstimator); + int nVersionRequired, nVersionThatWrote; + unsigned int nFileBestSeenHeight, nFileHistoricalFirst, nFileHistoricalBest; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + + // Read fee estimates file into temporary variables so existing data + // structures aren't corrupted if there is an exception. + filein >> nFileBestSeenHeight; + + if (nVersionThatWrote < 149900) { + // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. + // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor + double tempDecay; + filein >> tempDecay; + if (tempDecay <= 0 || tempDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + std::vector<double> tempBuckets; + filein >> tempBuckets; + size_t tempNum = tempBuckets.size(); + if (tempNum <= 1 || tempNum > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::map<double, unsigned int> tempMap; + + std::unique_ptr<TxConfirmStats> tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); + tempFeeStats->Read(filein, nVersionThatWrote, tempNum); + // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + + tempMap.clear(); + for (unsigned int i = 0; i < tempBuckets.size(); i++) { + tempMap[tempBuckets[i]] = i; + } + } + else { // nVersionThatWrote >= 149900 + filein >> nFileHistoricalFirst >> nFileHistoricalBest; + if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { + throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); + } + std::vector<double> fileBuckets; + filein >> fileBuckets; + size_t numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); + fileShortStats->Read(filein, nVersionThatWrote, numBuckets); + fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + + // Fee estimates file parsed correctly + // Copy buckets from file and refresh our bucketmap + buckets = fileBuckets; + bucketMap.clear(); + for (unsigned int i = 0; i < buckets.size(); i++) { + bucketMap[buckets[i]] = i; + } + + // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap + delete feeStats; + delete shortStats; + delete longStats; + feeStats = fileFeeStats.release(); + shortStats = fileShortStats.release(); + longStats = fileLongStats.release(); + + nBestSeenHeight = nFileBestSeenHeight; + historicalFirst = nFileHistoricalFirst; + historicalBest = nFileHistoricalBest; + } + } + catch (const std::exception& e) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); + return false; + } + return true; +} + +void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { + int64_t startclear = GetTimeMicros(); + std::vector<uint256> txids; + pool.queryHashes(txids); + LOCK(cs_feeEstimator); + for (auto& txid : txids) { + removeTx(txid, false); + } + int64_t endclear = GetTimeMicros(); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %ld micros\n",txids.size(), endclear - startclear); } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) { CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { + for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { feeset.insert(bucketBoundary); } } diff --git a/src/policy/fees.h b/src/policy/fees.h index dd01c90c45..f527c85004 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -6,8 +6,10 @@ #define BITCOIN_POLICYESTIMATOR_H #include "amount.h" +#include "feerate.h" #include "uint256.h" #include "random.h" +#include "sync.h" #include <map> #include <string> @@ -17,6 +19,7 @@ class CAutoFile; class CFeeRate; class CTxMemPoolEntry; class CTxMemPool; +class TxConfirmStats; /** \class CBlockPolicyEstimator * The BlockPolicyEstimator is used for estimating the feerate needed @@ -39,161 +42,58 @@ class CTxMemPool; * within your desired 5 blocks. * * Here is a brief description of the implementation: - * When a transaction enters the mempool, we - * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of feerate - * paid in each bucket. Then we calculate how many blocks Y it took each - * transaction to be mined and we track an array of counters in each bucket - * for how long it to took transactions to get confirmed from 1 to a max of 25 - * and we increment all the counters from Y up to 25. This is because for any - * number Z>=Y the transaction was successfully mined within Z blocks. We - * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given feerate - * bucket and the total number that were confirmed in each number 1-25 blocks - * or less for any bucket. We save this history by keeping an exponentially - * decaying moving average of each one of these stats. Furthermore we also - * keep track of the number unmined (in mempool) transactions in each bucket - * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that feerate bucket when calculating - * an estimate for any number of confirmations below the number of blocks - * they've been outstanding. + * When a transaction enters the mempool, we track the height of the block chain + * at entry. All further calculations are conducted only on this set of "seen" + * transactions. Whenever a block comes in, we count the number of transactions + * in each bucket and the total amount of feerate paid in each bucket. Then we + * calculate how many blocks Y it took each transaction to be mined. We convert + * from a number of blocks to a number of periods Y' each encompassing "scale" + * blocks. This is is tracked in 3 different data sets each up to a maximum + * number of periods. Within each data set we have an array of counters in each + * feerate bucket and we increment all the counters from Y' up to max periods + * representing that a tx was successfullly confirmed in less than or equal to + * that many periods. We want to save a history of this information, so at any + * time we have a counter of the total number of transactions that happened in a + * given feerate bucket and the total number that were confirmed in each of the + * periods or less for any bucket. We save this history by keeping an + * exponentially decaying moving average of each one of these stats. This is + * done for a different decay in each of the 3 data sets to keep relevant data + * from different time horizons. Furthermore we also keep track of the number + * unmined (in mempool or left mempool without being included in a block) + * transactions in each bucket and for how many blocks they have been + * outstanding and use both of these numbers to increase the number of transactions + * we've seen in that feerate bucket when calculating an estimate for any number + * of confirmations below the number of blocks they've been outstanding. */ -/** - * We will instantiate an instance of this class to track transactions that were - * included in a block. We will lump transactions into a bucket according to their - * approximate feerate and then track how long it took for those txs to be included in a block - * - * The tracking of unconfirmed (mempool) transactions is completely independent of the - * historical tracking of transactions that have been confirmed in a block. - */ -class TxConfirmStats -{ -private: - //Define the buckets we will group transactions into - std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) - std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket - - // For each bucket X: - // Count the total # of txs in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector<int> curBlockTxCt; - - // Count the total # of txs confirmed within Y blocks in each bucket - // Track the historical moving average of theses totals over blocks - std::vector<std::vector<double> > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] - - // Sum the total feerate of all tx's in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> avg; - // and calculate the total for the current block to update the moving average - std::vector<double> curBlockVal; +/* Identifier for each of the 3 different TxConfirmStats which will track + * history over different time horizons. */ +enum FeeEstimateHorizon { + SHORT_HALFLIFE = 0, + MED_HALFLIFE = 1, + LONG_HALFLIFE = 2 +}; - // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg feerate per bucket +/* Used to return detailed information about a feerate bucket */ +struct EstimatorBucket +{ + double start = -1; + double end = -1; + double withinTarget = 0; + double totalConfirmed = 0; + double inMempool = 0; + double leftMempool = 0; +}; +/* Used to return detailed information about a fee estimate calculation */ +struct EstimationResult +{ + EstimatorBucket pass; + EstimatorBucket fail; double decay; - - // Mempool counts of outstanding transactions - // For each bucket X, track the number of transactions in the mempool - // that are unconfirmed for each possible confirmation value Y - std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket - std::vector<int> oldUnconfTxs; - -public: - /** - * Initialize the data structures. This is called by BlockPolicyEstimator's - * constructor with default values. - * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track - * @param decay how much to decay the historical moving average per block - */ - void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); - - /** Clear the state of the curBlock variables to start counting for the new block */ - void ClearCurrent(unsigned int nBlockHeight); - - /** - * Record a new transaction data point in the current block stats - * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val the feerate of the transaction - * @warning blocksToConfirm is 1-based and has to be >= 1 - */ - void Record(int blocksToConfirm, double val); - - /** Record a new transaction entering the mempool*/ - unsigned int NewTx(unsigned int nBlockHeight, double val); - - /** Remove a transaction from mempool tracking stats*/ - void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); - - /** Update our estimates by decaying our historical moving average and updating - with the data gathered from the current block */ - void UpdateMovingAverages(); - - /** - * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets - * to make sure we have enough data points) whose transactions still have sufficient likelihood - * of being confirmed within the target number of confirmations - * @param confTarget target number of confirmations - * @param sufficientTxVal required average number of transactions per block in a bucket range - * @param minSuccess the success probability we require - * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR - * return the highest feerate such that all lower values fail minSuccess - * @param nBlockHeight the current block height - */ - double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight); - - /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() { return confAvg.size(); } - - /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout); - - /** - * Read saved state of estimation data from a file and replace all internal data structures and - * variables with this state. - */ - void Read(CAutoFile& filein); + unsigned int scale; }; - - -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ -static const double SUFFICIENT_FEETXS = 1; - -// Minimum and Maximum values for tracking feerates -// The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we -// might ever want to track. Historically this has been 1000 since it was -// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it -// invalidates old estimates files. So leave it at 1000 unless it becomes -// necessary to lower it, and then lower it substantially. -static constexpr double MIN_BUCKET_FEERATE = 1000; -static const double MAX_BUCKET_FEERATE = 1e7; -static const double INF_FEERATE = MAX_MONEY; - -// We have to lump transactions into buckets based on feerate, but we want to be able -// to give accurate estimates over a large range of potential feerates -// Therefore it makes sense to exponentially space the buckets -/** Spacing of FeeRate buckets */ -static const double FEE_SPACING = 1.1; - /** * We want to be able to estimate feerates that are needed on tx's to be included in * a certain number of blocks. Every time a block is added to the best chain, this class records @@ -201,41 +101,101 @@ static const double FEE_SPACING = 1.1; */ class CBlockPolicyEstimator { +private: + /** Track confirm delays up to 12 blocks for short horizon */ + static constexpr unsigned int SHORT_BLOCK_PERIODS = 12; + static constexpr unsigned int SHORT_SCALE = 1; + /** Track confirm delays up to 48 blocks for medium horizon */ + static constexpr unsigned int MED_BLOCK_PERIODS = 24; + static constexpr unsigned int MED_SCALE = 2; + /** Track confirm delays up to 1008 blocks for long horizon */ + static constexpr unsigned int LONG_BLOCK_PERIODS = 42; + static constexpr unsigned int LONG_SCALE = 24; + /** Historical estimates that are older than this aren't valid */ + static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; + + /** Decay of .962 is a half-life of 18 blocks or about 3 hours */ + static constexpr double SHORT_DECAY = .962; + /** Decay of .998 is a half-life of 144 blocks or about 1 day */ + static constexpr double MED_DECAY = .9952; + /** Decay of .9995 is a half-life of 1008 blocks or about 1 week */ + static constexpr double LONG_DECAY = .99931; + + /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ + static constexpr double HALF_SUCCESS_PCT = .6; + /** Require greater than 85% of X feerate transactions to be confirmed within Y blocks*/ + static constexpr double SUCCESS_PCT = .85; + /** Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks*/ + static constexpr double DOUBLE_SUCCESS_PCT = .95; + + /** Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance */ + static constexpr double SUFFICIENT_FEETXS = 0.1; + /** Require an avg of 0.5 tx when using short decay since there are fewer blocks considered*/ + static constexpr double SUFFICIENT_TXS_SHORT = 0.5; + + /** Minimum and Maximum values for tracking feerates + * The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we + * might ever want to track. Historically this has been 1000 since it was + * inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + * invalidates old estimates files. So leave it at 1000 unless it becomes + * necessary to lower it, and then lower it substantially. + */ + static constexpr double MIN_BUCKET_FEERATE = 1000; + static constexpr double MAX_BUCKET_FEERATE = 1e7; + + /** Spacing of FeeRate buckets + * We have to lump transactions into buckets based on feerate, but we want to be able + * to give accurate estimates over a large range of potential feerates + * Therefore it makes sense to exponentially space the buckets + */ + static constexpr double FEE_SPACING = 1.05; + public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); + ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries); - /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); - /** Process a transaction accepted to the mempool*/ void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); + bool removeTx(uint256 hash, bool inBlock); + + /** DEPRECATED. Return a feerate estimate */ + CFeeRate estimateFee(int confTarget) const; - /** Return a feerate estimate */ - CFeeRate estimateFee(int confTarget); + /** Estimate feerate needed to get be included in a block within confTarget + * blocks. If no answer can be given at confTarget, return an estimate at + * the closest target where one can be given. 'conservative' estimates are + * valid over longer time horizons also. + */ + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative = true) const; - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + /** Return a specific fee estimate calculation with a given success + * threshold and time horizon, and optionally return detailed data about + * calculation */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; /** Write estimation data to a file */ - void Write(CAutoFile& fileout); + bool Write(CAutoFile& fileout) const; /** Read estimation data from a file */ - void Read(CAutoFile& filein, int nFileVersion); + bool Read(CAutoFile& filein); + + /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ + void FlushUnconfirmed(CTxMemPool& pool); private: - CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main unsigned int nBestSeenHeight; + unsigned int firstRecordedHeight; + unsigned int historicalFirst; + unsigned int historicalBest; + struct TxStatsInfo { unsigned int blockHeight; @@ -247,14 +207,43 @@ private: std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats; + TxConfirmStats* feeStats; + TxConfirmStats* shortStats; + TxConfirmStats* longStats; unsigned int trackedTxs; unsigned int untrackedTxs; + + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + mutable CCriticalSection cs_feeEstimator; + + /** Process a transaction confirmed in a block*/ + bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + + /** Helper for estimateSmartFee */ + double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const; + /** Helper for estimateSmartFee */ + double estimateConservativeFee(unsigned int doubleTarget) const; + /** Number of blocks of data recorded while fee estimates have been running */ + unsigned int BlockSpan() const; + /** Number of blocks of recorded fee estimate data represented in saved data file */ + unsigned int HistoricalBlockSpan() const; + /** Calculation of highest target that reasonable estimate can be provided for */ + unsigned int MaxUsableEstimate() const; }; class FeeFilterRounder { +private: + static constexpr double MAX_FILTER_FEERATE = 1e7; + /** FEE_FILTER_SPACING is just used to provide some quantization of fee + * filter results. Historically it reused FEE_SPACING, but it is completely + * unrelated, and was made a separate constant so the two concepts are not + * tied together */ + static constexpr double FEE_FILTER_SPACING = 1.1; + public: /** Create new FeeFilterRounder */ FeeFilterRounder(const CFeeRate& minIncrementalFee); @@ -266,4 +255,5 @@ private: std::set<double> feeset; FastRandomContext insecure_rand; }; + #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index ec398f6627..2b19a6714b 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -8,12 +8,50 @@ #include "policy/policy.h" #include "validation.h" +#include "coins.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" #include <boost/foreach.hpp> +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee) +{ + // "Dust" is defined in terms of dustRelayFee, + // which has units satoshis-per-kilobyte. + // If you'd pay more than 1/3 in fees + // to spend something, then we consider it dust. + // A typical spendable non-segwit txout is 34 bytes big, and will + // need a CTxIn of at least 148 bytes to spend: + // so dust is a spendable txout less than + // 546*dustRelayFee/1000 (in satoshis). + // A typical spendable segwit txout is 31 bytes big, and will + // need a CTxIn of at least 67 bytes to spend: + // so dust is a spendable txout less than + // 294*dustRelayFee/1000 (in satoshis). + if (txout.scriptPubKey.IsUnspendable()) + return 0; + + size_t nSize = GetSerializeSize(txout, SER_DISK, 0); + int witnessversion = 0; + std::vector<unsigned char> witnessprogram; + + if (txout.scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { + // sum the sizes of the parts of a transaction input + // with 75% segwit discount applied to the script size. + nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); + } else { + nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above + } + + return 3 * dustRelayFee.GetFee(nSize); +} + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee) +{ + return (txout.nValue < GetDustThreshold(txout, dustRelayFee)); +} + /** * Check transaction inputs to mitigate two * potential denial-of-service attacks: @@ -105,7 +143,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnes else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { reason = "bare-multisig"; return false; - } else if (txout.IsDust(dustRelayFee)) { + } else if (IsDust(txout, ::dustRelayFee)) { reason = "dust"; return false; } diff --git a/src/policy/policy.h b/src/policy/policy.h index 6df541bc0f..2c2ea9d5b8 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -7,12 +7,14 @@ #define BITCOIN_POLICY_POLICY_H #include "consensus/consensus.h" +#include "feerate.h" #include "script/interpreter.h" #include "script/standard.h" #include <string> class CCoinsViewCache; +class CTxOut; /** Default for -blockmaxsize, which controls the maximum size of block the mining code will create **/ static const unsigned int DEFAULT_BLOCK_MAX_SIZE = 750000; @@ -72,6 +74,10 @@ static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_ static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST; +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); + bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled = false); /** * Check for standard transaction types diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index d413e8b087..00ac0b92b5 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -22,8 +22,8 @@ public: uint256 hash; uint32_t n; - COutPoint() { SetNull(); } - COutPoint(uint256 hashIn, uint32_t nIn) { hash = hashIn; n = nIn; } + COutPoint(): n((uint32_t) -1) { } + COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } ADD_SERIALIZE_METHODS; @@ -161,43 +161,6 @@ public: return (nValue == -1); } - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const - { - // "Dust" is defined in terms of CTransaction::minRelayTxFee, - // which has units satoshis-per-kilobyte. - // If you'd pay more than 1/3 in fees - // to spend something, then we consider it dust. - // A typical spendable non-segwit txout is 34 bytes big, and will - // need a CTxIn of at least 148 bytes to spend: - // so dust is a spendable txout less than - // 546*minRelayTxFee/1000 (in satoshis). - // A typical spendable segwit txout is 31 bytes big, and will - // need a CTxIn of at least 67 bytes to spend: - // so dust is a spendable txout less than - // 294*minRelayTxFee/1000 (in satoshis). - if (scriptPubKey.IsUnspendable()) - return 0; - - size_t nSize = GetSerializeSize(*this, SER_DISK, 0); - int witnessversion = 0; - std::vector<unsigned char> witnessprogram; - - if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { - // sum the sizes of the parts of a transaction input - // with 75% segwit discount applied to the script size. - nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); - } else { - nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above - } - - return 3 * minRelayTxFee.GetFee(nSize); - } - - bool IsDust(const CFeeRate &minRelayTxFee) const - { - return (nValue < GetDustThreshold(minRelayTxFee)); - } - friend bool operator==(const CTxOut& a, const CTxOut& b) { return (a.nValue == b.nValue && diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 4b34e73eb7..f8a99506c1 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -181,7 +181,5 @@ void BanTableModel::sort(int column, Qt::SortOrder order) bool BanTableModel::shouldShow() { - if (priv->size() > 0) - return true; - return false; + return priv->size() > 0; } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index be79a67990..5c26baef9e 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -478,6 +478,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); connect(_clientModel, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); + modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); setNumBlocks(_clientModel->getNumBlocks(), _clientModel->getLastBlockDate(), _clientModel->getVerificationProgress(NULL), false); connect(_clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); @@ -505,8 +506,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // initialize the disable state of the tray icon with the current value in the model. setTrayIconVisible(optionsModel->getHideTrayIcon()); } - - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); } else { // Disable possibility to show main window via action toggleHideAction->setEnabled(false); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb10e49422..de00eacdb9 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -9,6 +9,7 @@ #include "guiutil.h" #include "peertablemodel.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "clientversion.h" @@ -36,6 +37,8 @@ ClientModel::ClientModel(OptionsModel *_optionsModel, QObject *parent) : banTableModel(0), pollTimer(0) { + cachedBestHeaderHeight = -1; + cachedBestHeaderTime = -1; peerTableModel = new PeerTableModel(this); banTableModel = new BanTableModel(this); pollTimer = new QTimer(this); @@ -74,18 +77,28 @@ int ClientModel::getNumBlocks() const int ClientModel::getHeaderTipHeight() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->nHeight; + if (cachedBestHeaderHeight == -1) { + // make sure we initially populate the cache via a cs_main lock + // otherwise we need to wait for a tip update + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderHeight; } int64_t ClientModel::getHeaderTipTime() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->GetBlockTime(); + if (cachedBestHeaderTime == -1) { + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderTime; } quint64 ClientModel::getTotalBytesRecv() const @@ -283,6 +296,11 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CB int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (fHeader) { + // cache best headers time and height to reduce future cs_main locks + clientmodel->cachedBestHeaderHeight = pIndex->nHeight; + clientmodel->cachedBestHeaderTime = pIndex->GetBlockTime(); + } // if we are in-sync, update the UI regardless of last update time if (!initialSync || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { //pass a async signal to the UI thread diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 2c10e633b8..4c92e2144e 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -8,6 +8,8 @@ #include <QObject> #include <QDateTime> +#include <atomic> + class AddressTableModel; class BanTableModel; class OptionsModel; @@ -81,6 +83,10 @@ public: QString formatClientStartupTime() const; QString dataDir() const; + // caches for the best header + mutable std::atomic<int> cachedBestHeaderHeight; + mutable std::atomic<int64_t> cachedBestHeaderTime; + private: OptionsModel *optionsModel; PeerTableModel *peerTableModel; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 1d19c65753..2a331d4fae 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -15,6 +15,7 @@ #include "wallet/coincontrol.h" #include "init.h" +#include "policy/fees.h" #include "policy/policy.h" #include "validation.h" // For mempool #include "wallet/wallet.h" @@ -433,8 +434,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) { CTxOut txout(amount, (CScript)std::vector<unsigned char>(24, 0)); txDummy.vout.push_back(txout); - if (txout.IsDust(dustRelayFee)) - fDust = true; + fDust |= IsDust(txout, ::dustRelayFee); } } @@ -512,7 +512,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) nPayFee = coinControl->nMinimumTotalFee; @@ -526,10 +526,10 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (nChange > 0 && nChange < MIN_CHANGE) { CTxOut txout(nChange, (CScript)std::vector<unsigned char>(24, 0)); - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (CoinControlDialog::fSubtractFeeFromAmount) // dust-change will be raised until no dust - nChange = txout.GetDustThreshold(dustRelayFee); + nChange = GetDustThreshold(txout, ::dustRelayFee); else { nPayFee += nChange; @@ -592,7 +592,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (payTxFee.GetFeePerK() > 0) dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), payTxFee.GetFeePerK()) / 1000; else { - dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), mempool.estimateSmartFee(nTxConfirmTarget).GetFeePerK()) / 1000; + dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(nTxConfirmTarget, NULL, ::mempool).GetFeePerK()) / 1000; } QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 0b29201872..0f1b3f4a73 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -692,17 +692,34 @@ <item> <layout class="QHBoxLayout" name="horizontalLayout_Buttons"> <item> - <widget class="QPushButton" name="resetButton"> - <property name="toolTip"> - <string>Reset all client options to default.</string> - </property> - <property name="text"> - <string>&Reset Options</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> + <layout class="QVBoxLayout" name="verticalLayout_Buttons"> + <item> + <widget class="QPushButton" name="openBitcoinConfButton"> + <property name="toolTip"> + <string>Open the %1 configuration file from the working directory.</string> + </property> + <property name="text"> + <string>Open Configuration File</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="resetButton"> + <property name="toolTip"> + <string>Reset all client options to default.</string> + </property> + <property name="text"> + <string>&Reset Options</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> </item> <item> <spacer name="horizontalSpacer_1"> @@ -756,27 +773,48 @@ </spacer> </item> <item> - <widget class="QPushButton" name="okButton"> - <property name="text"> - <string>&OK</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - <property name="default"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="cancelButton"> - <property name="text"> - <string>&Cancel</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> + <layout class="QVBoxLayout" name="verticalLayout_4"> + <item> + <spacer name="verticalSpacer"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>40</height> + </size> + </property> + </spacer> + </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <widget class="QPushButton" name="okButton"> + <property name="text"> + <string>&OK</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + <property name="default"> + <bool>true</bool> + </property> + </widget> + </item> + <item> + <widget class="QPushButton" name="cancelButton"> + <property name="text"> + <string>&Cancel</string> + </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + </widget> + </item> + </layout> + </item> + </layout> </item> </layout> </item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index cc183908d4..52256ca5c4 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -1235,7 +1235,7 @@ <bool>false</bool> </property> <property name="default"> - <bool>true</bool> + <bool>false</bool> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index a2699d374a..bffa81137b 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -251,7 +251,7 @@ bool isDust(const QString& address, const CAmount& amount) CTxDestination dest = CBitcoinAddress(address.toStdString()).Get(); CScript script = GetScriptForDestination(dest); CTxOut txOut(amount, script); - return txOut.IsDust(dustRelayFee); + return IsDust(txOut, ::dustRelayFee); } QString HtmlEscape(const QString& str, bool fMultiLine) @@ -415,6 +415,22 @@ void openDebugLogfile() QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathDebug))); } +bool openBitcoinConf() +{ + boost::filesystem::path pathConfig = GetConfigFile(BITCOIN_CONF_FILENAME); + + /* Create the file */ + boost::filesystem::ofstream configFile(pathConfig, std::ios_base::app); + + if (!configFile.good()) + return false; + + configFile.close(); + + /* Open bitcoin.conf with the associated application */ + return QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathConfig))); +} + void SubstituteFonts(const QString& language) { #if defined(Q_OS_MAC) @@ -839,14 +855,17 @@ void restoreWindowGeometry(const QString& strSetting, const QSize& defaultSize, QPoint pos = settings.value(strSetting + "Pos").toPoint(); QSize size = settings.value(strSetting + "Size", defaultSize).toSize(); - if (!pos.x() && !pos.y()) { - QRect screen = QApplication::desktop()->screenGeometry(); - pos.setX((screen.width() - size.width()) / 2); - pos.setY((screen.height() - size.height()) / 2); - } - parent->resize(size); parent->move(pos); + + if ((!pos.x() && !pos.y()) || (QApplication::desktop()->screenNumber(parent) == -1)) + { + QRect screen = QApplication::desktop()->screenGeometry(); + QPoint defaultPos((screen.width() - defaultSize.width()) / 2, + (screen.height() - defaultSize.height()) / 2); + parent->resize(defaultSize); + parent->move(defaultPos); + } } void setClipboard(const QString& str) diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index f956770156..d6aa8c4ea6 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -113,6 +113,9 @@ namespace GUIUtil // Open debug.log void openDebugLogfile(); + // Open the config file + bool openBitcoinConf(); + // Replace invalid default fonts with known good ones void SubstituteFonts(const QString& language); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 7ff00b1e9e..efb25aaf18 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -232,6 +232,18 @@ void OptionsDialog::on_resetButton_clicked() } } +void OptionsDialog::on_openBitcoinConfButton_clicked() +{ + /* explain the purpose of the config file */ + QMessageBox::information(this, tr("Configuration options"), + tr("The configuration file is used to specify advanced user options which override GUI settings. " + "Additionally, any command-line options will override this configuration file.")); + + /* show an error if there was some problem opening the file */ + if (!GUIUtil::openBitcoinConf()) + QMessageBox::critical(this, tr("Error"), tr("The configuration file could not be opened.")); +} + void OptionsDialog::on_okButton_clicked() { mapper->submit(); diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index d98c1dc193..f9f5823c05 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -47,6 +47,7 @@ private Q_SLOTS: /* set OK button state (enabled / disabled) */ void setOkButtonState(bool fState); void on_resetButton_clicked(); + void on_openBitcoinConfButton_clicked(); void on_okButton_clicked(); void on_cancelButton_clicked(); diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index dd75f12076..c31a7a478d 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -219,14 +219,16 @@ void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) if (GUIUtil::parseBitcoinURI(arg, &r) && !r.address.isEmpty()) { CBitcoinAddress address(r.address.toStdString()); + auto tempChainParams = CreateChainParams(CBaseChainParams::MAIN); - if (address.IsValid(Params(CBaseChainParams::MAIN))) + if (address.IsValid(*tempChainParams)) { SelectParams(CBaseChainParams::MAIN); } - else if (address.IsValid(Params(CBaseChainParams::TESTNET))) - { - SelectParams(CBaseChainParams::TESTNET); + else { + tempChainParams = CreateChainParams(CBaseChainParams::TESTNET); + if (address.IsValid(*tempChainParams)) + SelectParams(CBaseChainParams::TESTNET); } } } @@ -580,7 +582,7 @@ bool PaymentServer::processPaymentRequest(const PaymentRequestPlus& request, Sen // Extract and check amounts CTxOut txOut(sendingTo.second, sendingTo.first); - if (txOut.IsDust(dustRelayFee)) { + if (IsDust(txOut, ::dustRelayFee)) { Q_EMIT message(tr("Payment request error"), tr("Requested payment amount of %1 is too small (considered dust).") .arg(BitcoinUnits::formatWithUnit(optionsModel->getDisplayUnit(), sendingTo.second)), CClientUIInterface::MSG_ERROR); diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 974a71ddca..fff072fd4c 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -166,7 +166,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const switch(index.column()) { case NetNodeId: - return rec->nodeStats.nodeid; + return (qint64)rec->nodeStats.nodeid; case Address: return QString::fromStdString(rec->nodeStats.addrName); case Subversion: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 5167232d6a..b200cb1127 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -98,7 +98,7 @@ class QtRPCTimerBase: public QObject, public RPCTimerBase { Q_OBJECT public: - QtRPCTimerBase(boost::function<void(void)>& _func, int64_t millis): + QtRPCTimerBase(std::function<void(void)>& _func, int64_t millis): func(_func) { timer.setSingleShot(true); @@ -110,7 +110,7 @@ private Q_SLOTS: void timeout() { func(); } private: QTimer timer; - boost::function<void(void)> func; + std::function<void(void)> func; }; class QtRPCTimerInterface: public RPCTimerInterface @@ -118,7 +118,7 @@ class QtRPCTimerInterface: public RPCTimerInterface public: ~QtRPCTimerInterface() {} const char *Name() { return "Qt"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new QtRPCTimerBase(func, millis); } @@ -730,8 +730,14 @@ void RPCConsole::clear(bool clearHistory) ).arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize)) ); +#ifdef Q_OS_MAC + QString clsKey = "(⌘)-L"; +#else + QString clsKey = "Ctrl-L"; +#endif + message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + - tr("Use up and down arrows to navigate history, and <b>Ctrl-L</b> to clear screen.") + "<br>" + + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + tr("Type <b>help</b> for an overview of available commands.")) + "<br><span class=\"secwarning\">" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramification of a command.") + @@ -829,7 +835,7 @@ void RPCConsole::on_lineEdit_returnPressed() cmdBeforeBrowsing = QString(); - message(CMD_REQUEST, cmd); + message(CMD_REQUEST, QString::fromStdString(strFilteredCmd)); Q_EMIT cmdRequest(cmd); cmd = QString::fromStdString(strFilteredCmd); @@ -1118,7 +1124,7 @@ void RPCConsole::disconnectSelectedNode() for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Find the node, disconnect it and clear the selected node if(g_connman->DisconnectNode(id)) clearSelectedNode(); @@ -1135,7 +1141,7 @@ void RPCConsole::banSelectedNode(int bantime) for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index ed7eab03f3..8dcb0fd016 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,6 +21,7 @@ #include "validation.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" +#include "policy/fees.h" #include "wallet/wallet.h" #include <QFontMetrics> @@ -30,8 +31,6 @@ #include <QTextDocument> #include <QTimer> -#define SEND_CONFIRM_DELAY 3 - SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::SendCoinsDialog), @@ -114,7 +113,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); - ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); } @@ -175,6 +173,9 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); updateGlobalFeeVariables(); + // set default rbf checkbox state + ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() == 0) @@ -660,7 +661,7 @@ void SendCoinsDialog::updateSmartFeeLabel() int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; int estimateFoundAtBlocks = nBlocksToConfirm; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks, ::mempool); if (feeRate <= CFeeRate(0)) // not enough data => minfee { ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index a402edc961..a932f129be 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -100,13 +100,14 @@ Q_SIGNALS: }; +#define SEND_CONFIRM_DELAY 3 class SendConfirmationDialog : public QMessageBox { Q_OBJECT public: - SendConfirmationDialog(const QString &title, const QString &text, int secDelay = 0, QWidget *parent = 0); + SendConfirmationDialog(const QString &title, const QString &text, int secDelay = SEND_CONFIRM_DELAY, QWidget *parent = 0); int exec(); private Q_SLOTS: diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index d8bcfedb7c..cae18f41a5 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -31,6 +31,9 @@ Q_IMPORT_PLUGIN(qjpcodecs) Q_IMPORT_PLUGIN(qtwcodecs) Q_IMPORT_PLUGIN(qkrcodecs) #else +#if defined(QT_QPA_PLATFORM_MINIMAL) +Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin); +#endif #if defined(QT_QPA_PLATFORM_XCB) Q_IMPORT_PLUGIN(QXcbIntegrationPlugin); #elif defined(QT_QPA_PLATFORM_WINDOWS) @@ -53,6 +56,11 @@ int main(int argc, char *argv[]) bool fInvalid = false; + // Prefer the "minimal" platform for the test instead of the normal default + // platform ("xcb", "windows", or "cocoa") so tests can't unintentially + // interfere with any background GUIs and don't require extra resources. + setenv("QT_QPA_PLATFORM", "minimal", 0); + // Don't remove this, it's needed to access // QApplication:: and QCoreApplication:: in the tests QApplication app(argc, argv); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index b7c1e4c4d5..a0dce3d997 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -68,13 +68,26 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) } //! Simple qt wallet tests. +// +// Test widgets can be debugged interactively calling show() on them and +// manually running the event loop, e.g.: +// +// sendCoinsDialog.show(); +// QEventLoop().exec(); +// +// This also requires overriding the default minimal Qt platform: +// +// src/qt/test/test_bitcoin-qt -platform xcb # Linux +// src/qt/test/test_bitcoin-qt -platform windows # Windows +// src/qt/test/test_bitcoin-qt -platform cocoa # macOS void WalletTests::walletTests() { // Set up wallet and chain with 101 blocks (1 mature block for spending). TestChain100Setup test; test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); bitdb.MakeMock(); - CWallet wallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + CWallet wallet(std::move(dbw)); bool firstRun; wallet.LoadWallet(firstRun); { diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index a9d9b6887e..4bb260aa58 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -18,14 +18,8 @@ */ bool TransactionRecord::showTransaction(const CWalletTx &wtx) { - if (wtx.IsCoinBase()) - { - // Ensures we show generated coins / mined transactions at depth 1 - if (!wtx.IsInMainChain()) - { - return false; - } - } + // There are currently no cases where we hide transactions, but + // we may want to use this in the future for things like RBF. return true; } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 30f4db9450..9008c81634 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -11,6 +11,7 @@ #include "guiutil.h" #include "optionsmodel.h" #include "platformstyle.h" +#include "sendcoinsdialog.h" #include "transactiondescdialog.h" #include "transactionfilterproxy.h" #include "transactionrecord.h" @@ -37,7 +38,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), model(0), transactionProxyModel(0), - transactionView(0), abandonAction(0), columnResizingFixer(0) + transactionView(0), abandonAction(0), bumpFeeAction(0), columnResizingFixer(0) { // Build filter row setContentsMargins(0,0,0,0); @@ -138,6 +139,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa // Actions abandonAction = new QAction(tr("Abandon transaction"), this); + bumpFeeAction = new QAction(tr("Increase transaction fee"), this); QAction *copyAddressAction = new QAction(tr("Copy address"), this); QAction *copyLabelAction = new QAction(tr("Copy label"), this); QAction *copyAmountAction = new QAction(tr("Copy amount"), this); @@ -156,6 +158,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa contextMenu->addAction(copyTxPlainText); contextMenu->addAction(showDetailsAction); contextMenu->addSeparator(); + contextMenu->addAction(bumpFeeAction); contextMenu->addAction(abandonAction); contextMenu->addAction(editLabelAction); @@ -173,6 +176,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SIGNAL(doubleClicked(QModelIndex))); connect(view, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(contextualMenu(QPoint))); + connect(bumpFeeAction, SIGNAL(triggered()), this, SLOT(bumpFee())); connect(abandonAction, SIGNAL(triggered()), this, SLOT(abandonTx())); connect(copyAddressAction, SIGNAL(triggered()), this, SLOT(copyAddress())); connect(copyLabelAction, SIGNAL(triggered()), this, SLOT(copyLabel())); @@ -372,6 +376,7 @@ void TransactionView::contextualMenu(const QPoint &point) uint256 hash; hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->transactionCanBeAbandoned(hash)); + bumpFeeAction->setEnabled(model->transactionSignalsRBF(hash)); if(index.isValid()) { @@ -397,6 +402,24 @@ void TransactionView::abandonTx() model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); } +void TransactionView::bumpFee() +{ + if(!transactionView || !transactionView->selectionModel()) + return; + QModelIndexList selection = transactionView->selectionModel()->selectedRows(0); + + // get the hash from the TxHashRole (QVariant / QString) + uint256 hash; + QString hashQStr = selection.at(0).data(TransactionTableModel::TxHashRole).toString(); + hash.SetHex(hashQStr.toStdString()); + + // Bump tx fee over the walletModel + if (model->bumpFee(hash)) { + // Update the table + model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); + } +} + void TransactionView::copyAddress() { GUIUtil::copyEntryData(transactionView, 0, TransactionTableModel::AddressRole); diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 595701cdd9..52e57cae4c 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -76,6 +76,7 @@ private: QDateTimeEdit *dateFrom; QDateTimeEdit *dateTo; QAction *abandonAction; + QAction *bumpFeeAction; QWidget *createDateRangeWidget(); @@ -99,6 +100,7 @@ private Q_SLOTS: void openThirdPartyTxUrl(QString url); void updateWatchOnlyColumn(bool fHaveWatchOnly); void abandonTx(); + void bumpFee(); Q_SIGNALS: void doubleClicked(const QModelIndex&); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ebcac53c25..37af8abb38 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -8,23 +8,29 @@ #include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" +#include "optionsmodel.h" #include "paymentserver.h" #include "recentrequeststablemodel.h" +#include "sendcoinsdialog.h" #include "transactiontablemodel.h" #include "base58.h" +#include "chain.h" #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman +#include "policy/rbf.h" #include "sync.h" #include "ui_interface.h" #include "util.h" // for GetBoolArg +#include "wallet/feebumper.h" #include "wallet/wallet.h" #include "wallet/walletdb.h" // for BackupWallet #include <stdint.h> #include <QDebug> +#include <QMessageBox> #include <QSet> #include <QTimer> @@ -692,6 +698,86 @@ bool WalletModel::abandonTransaction(uint256 hash) const return wallet->AbandonTransaction(hash); } +bool WalletModel::transactionSignalsRBF(uint256 hash) const +{ + LOCK2(cs_main, wallet->cs_wallet); + const CWalletTx *wtx = wallet->GetWalletTx(hash); + if (wtx && SignalsOptInRBF(*wtx)) + return true; + return false; +} + +bool WalletModel::bumpFee(uint256 hash) +{ + std::unique_ptr<CFeeBumper> feeBump; + { + LOCK2(cs_main, wallet->cs_wallet); + feeBump.reset(new CFeeBumper(wallet, hash, nTxConfirmTarget, false, 0, true)); + } + if (feeBump->getResult() != BumpFeeResult::OK) + { + QMessageBox::critical(0, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + + (feeBump->getErrors().size() ? QString::fromStdString(feeBump->getErrors()[0]) : "") +")"); + return false; + } + + // allow a user based fee verification + QString questionString = tr("Do you want to increase the fee?"); + questionString.append("<br />"); + CAmount oldFee = feeBump->getOldFee(); + CAmount newFee = feeBump->getNewFee(); + questionString.append("<table style=\"text-align: left;\">"); + questionString.append("<tr><td>"); + questionString.append(tr("Current fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("Increase:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee - oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("New fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee)); + questionString.append("</td></tr></table>"); + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); + confirmationDialog.exec(); + QMessageBox::StandardButton retval = (QMessageBox::StandardButton)confirmationDialog.result(); + + // cancel sign&broadcast if users doesn't want to bump the fee + if (retval != QMessageBox::Yes) { + return false; + } + + WalletModel::UnlockContext ctx(requestUnlock()); + if(!ctx.isValid()) + { + return false; + } + + // sign bumped transaction + bool res = false; + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->signTransaction(wallet); + } + if (!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Can't sign transaction.")); + return false; + } + // commit the bumped transaction + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->commit(wallet); + } + if(!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + + QString::fromStdString(feeBump->getErrors()[0])+")"); + return false; + } + return true; +} + bool WalletModel::isWalletEnabled() { return !GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 78e45dc369..df5acaf684 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -207,6 +207,9 @@ public: bool transactionCanBeAbandoned(uint256 hash) const; bool abandonTransaction(uint256 hash) const; + bool transactionSignalsRBF(uint256 hash) const; + bool bumpFee(uint256 hash); + static bool isWalletEnabled(); bool hdEnabled() const; diff --git a/src/random.cpp b/src/random.cpp index 6bcd0a70ba..3b9df3edaa 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -16,6 +16,8 @@ #include <stdlib.h> #include <limits> +#include <chrono> +#include <thread> #ifndef WIN32 #include <sys/time.h> @@ -32,6 +34,8 @@ #include <sys/sysctl.h> #endif +#include <mutex> + #include <openssl/err.h> #include <openssl/rand.h> @@ -43,15 +47,22 @@ static void RandFailure() static inline int64_t GetPerformanceCounter() { - int64_t nCounter = 0; -#ifdef WIN32 - QueryPerformanceCounter((LARGE_INTEGER*)&nCounter); + // Read the hardware time stamp counter when available. + // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + return __rdtsc(); +#elif !defined(_MSC_VER) && defined(__i386__) + uint64_t r = 0; + __asm__ volatile ("rdtsc" : "=A"(r)); // Constrain the r variable to the eax:edx pair. + return r; +#elif !defined(_MSC_VER) && (defined(__x86_64__) || defined(__amd64__)) + uint64_t r1 = 0, r2 = 0; + __asm__ volatile ("rdtsc" : "=a"(r1), "=d"(r2)); // Constrain r1 to rax and r2 to rdx. + return (r2 << 32) | r1; #else - timeval t; - gettimeofday(&t, NULL); - nCounter = (int64_t)(t.tv_sec * 1000000 + t.tv_usec); + // Fall back to using C++11 clock (usually microsecond or nanosecond precision) + return std::chrono::high_resolution_clock::now().time_since_epoch().count(); #endif - return nCounter; } void RandAddSeed() @@ -192,6 +203,10 @@ void GetRandBytes(unsigned char* buf, int num) } } +static std::mutex cs_rng_state; +static unsigned char rng_state[32] = {0}; +static uint64_t rng_counter = 0; + void GetStrongRandBytes(unsigned char* out, int num) { assert(num <= 32); @@ -207,8 +222,17 @@ void GetStrongRandBytes(unsigned char* out, int num) GetOSRand(buf); hasher.Write(buf, 32); + // Combine with and update state + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + // Produce output - hasher.Finalize(buf); memcpy(out, buf, num); memory_cleanse(buf, 64); } @@ -240,26 +264,22 @@ uint256 GetRandHash() return hash; } -FastRandomContext::FastRandomContext(bool fDeterministic) +void FastRandomContext::RandomSeed() { - // The seed values have some unlikely fixed points which we avoid. - if (fDeterministic) { - Rz = Rw = 11; - } else { - uint32_t tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x9068ffffU); - Rz = tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x464fffffU); - Rw = tmp; - } + uint256 seed = GetRandHash(); + rng.SetKey(seed.begin(), 32); + requires_seed = false; +} + +FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +{ + rng.SetKey(seed.begin(), 32); } bool Random_SanityCheck() { + uint64_t start = GetPerformanceCounter(); + /* This does not measure the quality of randomness, but it does test that * OSRandom() overwrites all 32 bytes of the output given a maximum * number of tries. @@ -286,5 +306,25 @@ bool Random_SanityCheck() tries += 1; } while (num_overwritten < NUM_OS_RANDOM_BYTES && tries < MAX_TRIES); - return (num_overwritten == NUM_OS_RANDOM_BYTES); /* If this failed, bailed out after too many tries */ + if (num_overwritten != NUM_OS_RANDOM_BYTES) return false; /* If this failed, bailed out after too many tries */ + + // Check that GetPerformanceCounter increases at least during a GetOSRand() call + 1ms sleep. + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + uint64_t stop = GetPerformanceCounter(); + if (stop == start) return false; + + // We called GetPerformanceCounter. Use it as entropy. + RAND_add((const unsigned char*)&start, sizeof(start), 1); + RAND_add((const unsigned char*)&stop, sizeof(stop), 1); + + return true; +} + +FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +{ + if (!fDeterministic) { + return; + } + uint256 seed; + rng.SetKey(seed.begin(), 32); } diff --git a/src/random.h b/src/random.h index 0464bdce14..9551e1c461 100644 --- a/src/random.h +++ b/src/random.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H +#include "crypto/chacha20.h" +#include "crypto/common.h" #include "uint256.h" #include <stdint.h> @@ -33,17 +35,79 @@ void GetStrongRandBytes(unsigned char* buf, int num); * This class is not thread-safe. */ class FastRandomContext { +private: + bool requires_seed; + ChaCha20 rng; + + unsigned char bytebuf[64]; + int bytebuf_size; + + uint64_t bitbuf; + int bitbuf_size; + + void RandomSeed(); + + void FillByteBuffer() + { + if (requires_seed) { + RandomSeed(); + } + rng.Output(bytebuf, sizeof(bytebuf)); + bytebuf_size = sizeof(bytebuf); + } + + void FillBitBuffer() + { + bitbuf = rand64(); + bitbuf_size = 64; + } + public: - explicit FastRandomContext(bool fDeterministic=false); + explicit FastRandomContext(bool fDeterministic = false); + + /** Initialize with explicit seed (only for testing) */ + explicit FastRandomContext(const uint256& seed); + + /** Generate a random 64-bit integer. */ + uint64_t rand64() + { + if (bytebuf_size < 8) FillByteBuffer(); + uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); + bytebuf_size -= 8; + return ret; + } - uint32_t rand32() { - Rz = 36969 * (Rz & 65535) + (Rz >> 16); - Rw = 18000 * (Rw & 65535) + (Rw >> 16); - return (Rw << 16) + Rz; + /** Generate a random (bits)-bit integer. */ + uint64_t randbits(int bits) { + if (bits == 0) { + return 0; + } else if (bits > 32) { + return rand64() >> (64 - bits); + } else { + if (bitbuf_size < bits) FillBitBuffer(); + uint64_t ret = bitbuf & (~(uint64_t)0 >> (64 - bits)); + bitbuf >>= bits; + bitbuf_size -= bits; + return ret; + } } - uint32_t Rz; - uint32_t Rw; + /** Generate a random integer in the range [0..range). */ + uint64_t randrange(uint64_t range) + { + --range; + int bits = CountBits(range); + while (true) { + uint64_t ret = randbits(bits); + if (ret <= range) return ret; + } + } + + /** Generate a random 32-bit integer. */ + uint32_t rand32() { return randbits(32); } + + /** Generate a random boolean. */ + bool randbool() { return randbits(1); } }; /* Number of random bytes returned by GetOSRand. diff --git a/src/rest.cpp b/src/rest.cpp index 9dcaf269d6..7537ed4502 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -5,6 +5,7 @@ #include "chain.h" #include "chainparams.h" +#include "core_io.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "validation.h" @@ -56,10 +57,6 @@ struct CCoin { } }; -/* Defined in rawtransaction.cpp */ -void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); - static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); @@ -383,7 +380,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, objTx); + TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -577,7 +574,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); utxo.push_back(Pair("scriptPubKey", o)); utxos.push_back(utxo); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 01066d0eb2..9d72a23e6d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -12,6 +12,8 @@ #include "coins.h" #include "consensus/validation.h" #include "validation.h" +#include "core_io.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" @@ -42,7 +44,6 @@ static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); double GetDifficulty(const CBlockIndex* blockindex) { @@ -123,7 +124,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, uint256(), objTx); + TxToUniv(*tx, uint256(), objTx); txs.push_back(objTx); } else @@ -412,6 +413,7 @@ UniValue getrawmempool(const JSONRPCRequest& request) throw std::runtime_error( "getrawmempool ( verbose )\n" "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" "\nArguments:\n" "1. verbose (boolean, optional, default=false) True for a json object, false for array of transaction ids\n" "\nResult: (for verbose = false):\n" @@ -686,13 +688,16 @@ UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "getblock \"blockhash\" ( verbose )\n" - "\nIf verbose is false, returns a string that is serialized, hex-encoded data for block 'hash'.\n" - "If verbose is true, returns an Object with information about block <hash>.\n" + "getblock \"blockhash\" ( verbosity ) \n" + "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" + "If verbosity is 1, returns an Object with information about block <hash>.\n" + "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n" "\nArguments:\n" "1. \"blockhash\" (string, required) The block hash\n" - "2. verbose (boolean, optional, default=true) true for a json object, false for the hex encoded data\n" - "\nResult (for verbose = true):\n" + "2. verbosity (numeric, optional, default=1) 0 for hex encoded data, 1 for a json object, and 2 for json object with transaction data\n" + "\nResult (for verbosity = 0):\n" + "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 1):\n" "{\n" " \"hash\" : \"hash\", (string) the block hash (same as provided)\n" " \"confirmations\" : n, (numeric) The number of confirmations, or -1 if the block is not on the main chain\n" @@ -716,8 +721,14 @@ UniValue getblock(const JSONRPCRequest& request) " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\" (string) The hash of the next block\n" "}\n" - "\nResult (for verbose=false):\n" - "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 2):\n" + "{\n" + " ..., Same output as verbosity = 1.\n" + " \"tx\" : [ (array of Objects) The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 \"tx\" result.\n" + " ,...\n" + " ],\n" + " ,... Same output as verbosity = 1.\n" + "}\n" "\nExamples:\n" + HelpExampleCli("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") @@ -728,9 +739,13 @@ UniValue getblock(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); - bool fVerbose = true; - if (request.params.size() > 1) - fVerbose = request.params[1].get_bool(); + int verbosity = 1; + if (request.params.size() > 1) { + if(request.params[1].isNum()) + verbosity = request.params[1].get_int(); + else + verbosity = request.params[1].get_bool() ? 1 : 0; + } if (mapBlockIndex.count(hash) == 0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -749,7 +764,7 @@ UniValue getblock(const JSONRPCRequest& request) // block). throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); - if (!fVerbose) + if (verbosity <= 0) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; @@ -757,7 +772,7 @@ UniValue getblock(const JSONRPCRequest& request) return strHex; } - return blockToJSON(block, pblockindex); + return blockToJSON(block, pblockindex, verbosity >= 2); } struct CCoinsStats @@ -975,7 +990,7 @@ UniValue gettxout(const JSONRPCRequest& request) ret.push_back(Pair("confirmations", pindex->nHeight - coins.nHeight + 1)); ret.push_back(Pair("value", ValueFromAmount(coins.vout[n].nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToUniv(coins.vout[n].scriptPubKey, o, true); ret.push_back(Pair("scriptPubKey", o)); ret.push_back(Pair("version", coins.nVersion)); ret.push_back(Pair("coinbase", coins.fCoinBase)); @@ -1415,13 +1430,79 @@ UniValue reconsiderblock(const JSONRPCRequest& request) return NullUniValue; } +UniValue getchaintxstats(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) + throw std::runtime_error( + "getchaintxstats ( nblocks blockhash )\n" + "\nCompute statistics about the total number and rate of transactions in the chain.\n" + "\nArguments:\n" + "1. nblocks (numeric, optional) Size of the window in number of blocks (default: one month).\n" + "2. \"blockhash\" (string, optional) The hash of the block that ends the window.\n" + "\nResult:\n" + "{\n" + " \"time\": xxxxx, (numeric) The timestamp for the statistics in UNIX format.\n" + " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" + " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getchaintxstats", "") + + HelpExampleRpc("getchaintxstats", "2016") + ); + + const CBlockIndex* pindex; + int blockcount = 30 * 24 * 60 * 60 / Params().GetConsensus().nPowTargetSpacing; // By default: 1 month + + if (request.params.size() > 0 && !request.params[0].isNull()) { + blockcount = request.params[0].get_int(); + } + + bool havehash = request.params.size() > 1 && !request.params[1].isNull(); + uint256 hash; + if (havehash) { + hash = uint256S(request.params[1].get_str()); + } + + { + LOCK(cs_main); + if (havehash) { + auto it = mapBlockIndex.find(hash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + pindex = it->second; + if (!chainActive.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); + } + } else { + pindex = chainActive.Tip(); + } + } + + if (blockcount < 1 || blockcount >= pindex->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 1 and the block's height"); + } + + const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + + UniValue ret(UniValue::VOBJ); + ret.push_back(Pair("time", (int64_t)pindex->nTime)); + ret.push_back(Pair("txcount", (int64_t)pindex->nChainTx)); + ret.push_back(Pair("txrate", ((double)nTxDiff) / nTimeDiff)); + + return ret; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} }, + { "blockchain", "getchaintxstats", &getchaintxstats, true, {"nblocks", "blockhash"} }, { "blockchain", "getbestblockhash", &getbestblockhash, true, {} }, { "blockchain", "getblockcount", &getblockcount, true, {} }, - { "blockchain", "getblock", &getblock, true, {"blockhash","verbose"} }, + { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity|verbose"} }, { "blockchain", "getblockhash", &getblockhash, true, {"height"} }, { "blockchain", "getblockheader", &getblockheader, true, {"blockhash","verbose"} }, { "blockchain", "getchaintips", &getchaintips, true, {} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 35bc5d6a82..a3a692c14d 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -77,8 +77,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 0, "minconf" }, { "listunspent", 1, "maxconf" }, { "listunspent", 2, "addresses" }, - { "getblock", 1, "verbose" }, + { "listunspent", 4, "query_options" }, + { "getblock", 1, "verbosity" }, { "getblockheader", 1, "verbose" }, + { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "getrawtransaction", 1, "verbose" }, { "createrawtransaction", 0, "inputs" }, @@ -106,6 +108,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getrawmempool", 0, "verbose" }, { "estimatefee", 0, "nblocks" }, { "estimatesmartfee", 0, "nblocks" }, + { "estimatesmartfee", 1, "conservative" }, + { "estimaterawfee", 0, "nblocks" }, + { "estimaterawfee", 1, "threshold" }, + { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "fee_delta" }, { "setban", 2, "bantime" }, { "setban", 3, "absolute" }, @@ -113,6 +119,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempoolancestors", 1, "verbose" }, { "getmempooldescendants", 1, "verbose" }, { "bumpfee", 1, "options" }, + { "logging", 0, "include" }, + { "logging", 1, "exclude" }, + { "disconnectnode", 1, "nodeid" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 7e5f0d608e..d744269df1 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -15,6 +15,7 @@ #include "validation.h" #include "miner.h" #include "net.h" +#include "policy/fees.h" #include "pow.h" #include "rpc/blockchain.h" #include "rpc/server.h" @@ -27,7 +28,6 @@ #include <stdint.h> #include <boost/assign/list_of.hpp> -#include <boost/shared_ptr.hpp> #include <univalue.h> @@ -95,7 +95,7 @@ UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(request.params.size() > 0 ? request.params[0].get_int() : 120, request.params.size() > 1 ? request.params[1].get_int() : -1); } -UniValue generateBlocks(boost::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) +UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript) { static const int nInnerLoopCount = 0x10000; int nHeightStart = 0; @@ -167,7 +167,7 @@ UniValue generate(const JSONRPCRequest& request) nMaxTries = request.params[1].get_int(); } - boost::shared_ptr<CReserveScript> coinbaseScript; + std::shared_ptr<CReserveScript> coinbaseScript; GetMainSignals().ScriptForMining(coinbaseScript); // If the keypool is exhausted, no script is returned at all. Catch this. @@ -208,7 +208,7 @@ UniValue generatetoaddress(const JSONRPCRequest& request) if (!address.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); - boost::shared_ptr<CReserveScript> coinbaseScript(new CReserveScript()); + std::shared_ptr<CReserveScript> coinbaseScript = std::make_shared<CReserveScript>(); coinbaseScript->reserveScript = GetScriptForDestination(address.Get()); return generateBlocks(coinbaseScript, nGenerate, nMaxTries, false); @@ -710,7 +710,7 @@ public: submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {} protected: - virtual void BlockChecked(const CBlock& block, const CValidationState& stateIn) { + void BlockChecked(const CBlock& block, const CValidationState& stateIn) override { if (block.GetHash() != hash) return; found = true; @@ -797,6 +797,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "estimatefee nblocks\n" + "\nDEPRECATED. Please use estimatesmartfee for more intelligent estimates." "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks. Uses virtual transaction size of transaction\n" "as defined in BIP 141 (witness data is discounted).\n" @@ -819,7 +820,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (nBlocks < 1) nBlocks = 1; - CFeeRate feeRate = mempool.estimateFee(nBlocks); + CFeeRate feeRate = ::feeEstimator.estimateFee(nBlocks); if (feeRate == CFeeRate(0)) return -1.0; @@ -828,16 +829,19 @@ UniValue estimatefee(const JSONRPCRequest& request) UniValue estimatesmartfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "estimatesmartfee nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "estimatesmartfee nblocks (conservative)\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid. Uses virtual transaction size as defined\n" "in BIP 141 (witness data is discounted).\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric)\n" + "2. conservative (bool, optional, default=true) Whether to return a more conservative estimate which\n" + " also satisfies a longer history. A conservative estimate potentially returns a higher\n" + " feerate and is more likely to be sufficient for the desired target, but is not as\n" + " responsive to short term drops in the prevailing fee market\n" "\nResult:\n" "{\n" " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" @@ -854,15 +858,102 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)); int nBlocks = request.params[0].get_int(); + bool conservative = true; + if (request.params.size() > 1 && !request.params[1].isNull()) { + RPCTypeCheckArgument(request.params[1], UniValue::VBOOL); + conservative = request.params[1].get_bool(); + } UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocks, &answerFound); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool, conservative); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; } +UniValue estimaterawfee(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + throw std::runtime_error( + "estimaterawfee nblocks (threshold horizon)\n" + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + " implementation of fee estimation. The parameters it can be called with\n" + " and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" + " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" + " lower buckets. Default: 0.95\n" + "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" + " Default: 1\n" + "\nResult:\n" + "{\n" + " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " }\n" + " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" + "}\n" + "\n" + "A negative feerate is returned if no answer can be given.\n" + "\nExample:\n" + + HelpExampleCli("estimaterawfee", "6 0.9 1") + ); + + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VNUM)(UniValue::VNUM), true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + int nBlocks = request.params[0].get_int(); + double threshold = 0.95; + if (!request.params[1].isNull()) + threshold = request.params[1].get_real(); + FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; + if (!request.params[2].isNull()) { + int horizonInt = request.params[2].get_int(); + if (horizonInt < 0 || horizonInt > 2) { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); + } else { + horizon = (FeeEstimateHorizon)horizonInt; + } + } + UniValue result(UniValue::VOBJ); + CFeeRate feeRate; + EstimationResult buckets; + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + + result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); + result.push_back(Pair("decay", buckets.decay)); + result.push_back(Pair("scale", (int)buckets.scale)); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("pass", passbucket)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("fail", failbucket)); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- @@ -876,7 +967,9 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, true, {"nblocks","address","maxtries"} }, { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, - { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} }, + { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, + + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 24c5eeffe9..1f973a0c18 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,9 +4,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "base58.h" +#include "chain.h" #include "clientversion.h" #include "init.h" #include "validation.h" +#include "httpserver.h" #include "net.h" #include "netbase.h" #include "rpc/blockchain.h" @@ -555,6 +557,73 @@ UniValue getmemoryinfo(const JSONRPCRequest& request) } } +uint32_t getCategoryMask(UniValue cats) { + cats = cats.get_array(); + uint32_t mask = 0; + for (unsigned int i = 0; i < cats.size(); ++i) { + uint32_t flag = 0; + std::string cat = cats[i].get_str(); + if (!GetLogCategory(&flag, &cat)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); + } + mask |= flag; + } + return mask; +} + +UniValue logging(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) { + throw std::runtime_error( + "logging [include,...] <exclude>\n" + "Gets and sets the logging configuration.\n" + "When called without an argument, returns the list of categories that are currently being debug logged.\n" + "When called with arguments, adds or removes categories from debug logging.\n" + "The valid logging categories are: " + ListLogCategories() + "\n" + "libevent logging is configured on startup and cannot be modified by this RPC during runtime." + "Arguments:\n" + "1. \"include\" (array of strings) add debug logging for these categories.\n" + "2. \"exclude\" (array of strings) remove debug logging for these categories.\n" + "\nResult: <categories> (string): a list of the logging categories that are active.\n" + "\nExamples:\n" + + HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + + HelpExampleRpc("logging", "[\"all\"], \"[libevent]\"") + ); + } + + uint32_t originalLogCategories = logCategories; + if (request.params.size() > 0 && request.params[0].isArray()) { + logCategories |= getCategoryMask(request.params[0]); + } + + if (request.params.size() > 1 && request.params[1].isArray()) { + logCategories &= ~getCategoryMask(request.params[1]); + } + + // Update libevent logging if BCLog::LIBEVENT has changed. + // If the library version doesn't allow it, UpdateHTTPServerLogging() returns false, + // in which case we should clear the BCLog::LIBEVENT flag. + // Throw an error if the user has explicitly asked to change only the libevent + // flag and it failed. + uint32_t changedLogCategories = originalLogCategories ^ logCategories; + if (changedLogCategories & BCLog::LIBEVENT) { + if (!UpdateHTTPServerLogging(logCategories & BCLog::LIBEVENT)) { + logCategories &= ~BCLog::LIBEVENT; + if (changedLogCategories == BCLog::LIBEVENT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "libevent logging cannot be updated when using libevent before v2.1.1."); + } + } + } + + UniValue result(UniValue::VOBJ); + std::vector<CLogCategoryActive> vLogCatActive = ListActiveLogCategories(); + for (const auto& logCatActive : vLogCatActive) { + result.pushKV(logCatActive.category, logCatActive.active); + } + + return result; +} + UniValue echo(const JSONRPCRequest& request) { if (request.fHelp) @@ -581,7 +650,8 @@ static const CRPCCommand commands[] = /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, true, {"timestamp"}}, { "hidden", "echo", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "echojson", &echo, true, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "logging", &logging, true, {"include", "exclude"}}, }; void RegisterMiscRPCCommands(CRPCTable &t) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 44c6e6d308..cde5ae723b 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -234,23 +234,43 @@ UniValue addnode(const JSONRPCRequest& request) UniValue disconnectnode(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() == 0 || request.params.size() >= 3) throw std::runtime_error( - "disconnectnode \"node\" \n" - "\nImmediately disconnects from the specified node.\n" + "disconnectnode \"[address]\" [nodeid]\n" + "\nImmediately disconnects from the specified peer node.\n" + "\nStrictly one out of 'address' and 'nodeid' can be provided to identify the node.\n" + "\nTo disconnect by nodeid, either set 'address' to the empty string, or call using the named 'nodeid' argument only.\n" "\nArguments:\n" - "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" + "1. \"address\" (string, optional) The IP address/port of the node\n" + "2. \"nodeid\" (number, optional) The node ID (see getpeerinfo for node IDs)\n" "\nExamples:\n" + HelpExampleCli("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleCli("disconnectnode", "\"\" 1") + HelpExampleRpc("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleRpc("disconnectnode", "\"\", 1") ); if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - bool ret = g_connman->DisconnectNode(request.params[0].get_str()); - if (!ret) + bool success; + const UniValue &address_arg = request.params[0]; + const UniValue &id_arg = request.params.size() < 2 ? NullUniValue : request.params[1]; + + if (!address_arg.isNull() && id_arg.isNull()) { + /* handle disconnect-by-address */ + success = g_connman->DisconnectNode(address_arg.get_str()); + } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { + /* handle disconnect-by-id */ + NodeId nodeid = (NodeId) id_arg.get_int64(); + success = g_connman->DisconnectNode(nodeid); + } else { + throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); + } + + if (!success) { throw JSONRPCError(RPC_CLIENT_NODE_NOT_CONNECTED, "Node not found in connected nodes"); + } return NullUniValue; } @@ -607,7 +627,7 @@ static const CRPCCommand commands[] = { "network", "ping", &ping, true, {} }, { "network", "getpeerinfo", &getpeerinfo, true, {} }, { "network", "addnode", &addnode, true, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, true, {"node"} }, + { "network", "disconnectnode", &disconnectnode, true, {"address", "nodeid"} }, { "network", "getaddednodeinfo", &getaddednodeinfo, true, {"node"} }, { "network", "getnettotals", &getnettotals, true, {} }, { "network", "getnetworkinfo", &getnetworkinfo, true, {} }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 717e9d75f3..683bb25246 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -34,77 +34,15 @@ #include <univalue.h> -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) -{ - txnouttype type; - std::vector<CTxDestination> addresses; - int nRequired; - - out.push_back(Pair("asm", ScriptToAsmStr(scriptPubKey))); - if (fIncludeHex) - out.push_back(Pair("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); - - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) { - out.push_back(Pair("type", GetTxnOutputType(type))); - return; - } - - out.push_back(Pair("reqSigs", nRequired)); - out.push_back(Pair("type", GetTxnOutputType(type))); - - UniValue a(UniValue::VARR); - BOOST_FOREACH(const CTxDestination& addr, addresses) - a.push_back(CBitcoinAddress(addr).ToString()); - out.push_back(Pair("addresses", a)); -} void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); - entry.push_back(Pair("hash", tx.GetWitnessHash().GetHex())); - entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); - entry.push_back(Pair("vsize", (int)::GetVirtualTransactionSize(tx))); - entry.push_back(Pair("version", tx.nVersion)); - entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); - - UniValue vin(UniValue::VARR); - for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxIn& txin = tx.vin[i]; - UniValue in(UniValue::VOBJ); - if (tx.IsCoinBase()) - in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - else { - in.push_back(Pair("txid", txin.prevout.hash.GetHex())); - in.push_back(Pair("vout", (int64_t)txin.prevout.n)); - UniValue o(UniValue::VOBJ); - o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); - o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - in.push_back(Pair("scriptSig", o)); - } - if (tx.HasWitness()) { - UniValue txinwitness(UniValue::VARR); - for (unsigned int j = 0; j < tx.vin[i].scriptWitness.stack.size(); j++) { - std::vector<unsigned char> item = tx.vin[i].scriptWitness.stack[j]; - txinwitness.push_back(HexStr(item.begin(), item.end())); - } - in.push_back(Pair("txinwitness", txinwitness)); - } - in.push_back(Pair("sequence", (int64_t)txin.nSequence)); - vin.push_back(in); - } - entry.push_back(Pair("vin", vin)); - UniValue vout(UniValue::VARR); - for (unsigned int i = 0; i < tx.vout.size(); i++) { - const CTxOut& txout = tx.vout[i]; - UniValue out(UniValue::VOBJ); - out.push_back(Pair("value", ValueFromAmount(txout.nValue))); - out.push_back(Pair("n", (int64_t)i)); - UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(txout.scriptPubKey, o, true); - out.push_back(Pair("scriptPubKey", o)); - vout.push_back(out); - } - entry.push_back(Pair("vout", vout)); + // Call into TxToUniv() in bitcoin-common to decode the transaction hex. + // + // Blockchain contextual information (confirmations and blocktime) is not + // available to code in bitcoin-common, so we query them here and push the + // data into the returned UniValue. + TxToUniv(tx, uint256(), entry); if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); @@ -525,7 +463,7 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); UniValue result(UniValue::VOBJ); - TxToJSON(CTransaction(std::move(mtx)), uint256(), result); + TxToUniv(CTransaction(std::move(mtx)), uint256(), result); return result; } @@ -565,7 +503,7 @@ UniValue decodescript(const JSONRPCRequest& request) } else { // Empty scripts are valid } - ScriptPubKeyToJSON(script, r, false); + ScriptPubKeyToUniv(script, r, false); UniValue type; type = find_value(r, "type"); @@ -585,6 +523,11 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: UniValue entry(UniValue::VOBJ); entry.push_back(Pair("txid", txin.prevout.hash.ToString())); entry.push_back(Pair("vout", (uint64_t)txin.prevout.n)); + UniValue witness(UniValue::VARR); + for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { + witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + } + entry.push_back(Pair("witness", witness)); entry.push_back(Pair("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); entry.push_back(Pair("sequence", (uint64_t)txin.nSequence)); entry.push_back(Pair("error", strMessage)); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 4032c5163c..c5fbff0077 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -22,6 +22,8 @@ #include <boost/signals2/signal.hpp> #include <boost/thread.hpp> #include <boost/algorithm/string/case_conv.hpp> // for to_upper() +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> #include <memory> // for unique_ptr #include <unordered_map> @@ -42,17 +44,17 @@ static struct CRPCSignals boost::signals2::signal<void (const CRPCCommand&)> PreCommand; } g_rpcSignals; -void RPCServer::OnStarted(boost::function<void ()> slot) +void RPCServer::OnStarted(std::function<void ()> slot) { g_rpcSignals.Started.connect(slot); } -void RPCServer::OnStopped(boost::function<void ()> slot) +void RPCServer::OnStopped(std::function<void ()> slot) { g_rpcSignals.Stopped.connect(slot); } -void RPCServer::OnPreCommand(boost::function<void (const CRPCCommand&)> slot) +void RPCServer::OnPreCommand(std::function<void (const CRPCCommand&)> slot) { g_rpcSignals.PreCommand.connect(boost::bind(slot, _1)); } @@ -369,9 +371,7 @@ void JSONRPCRequest::parse(const UniValue& valRequest) if (!valMethod.isStr()) throw JSONRPCError(RPC_INVALID_REQUEST, "Method must be a string"); strMethod = valMethod.get_str(); - if (strMethod != "getblocktemplate") { - LogPrint(BCLog::RPC, "ThreadRPCServer method=%s\n", SanitizeString(strMethod)); - } + LogPrint(BCLog::RPC, "ThreadRPCServer method=%s\n", SanitizeString(strMethod)); // Parse params UniValue valParams = find_value(request, "params"); @@ -434,8 +434,16 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c } // Process expected parameters. int hole = 0; - for (const std::string &argName: argNames) { - auto fr = argsIn.find(argName); + for (const std::string &argNamePattern: argNames) { + std::vector<std::string> vargNames; + boost::algorithm::split(vargNames, argNamePattern, boost::algorithm::is_any_of("|")); + auto fr = argsIn.end(); + for (const std::string & argName : vargNames) { + fr = argsIn.find(argName); + if (fr != argsIn.end()) { + break; + } + } if (fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, @@ -528,7 +536,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface) timerInterface = NULL; } -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds) +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds) { if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); diff --git a/src/rpc/server.h b/src/rpc/server.h index 34a9d3c24c..1e984cbc0d 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -15,8 +15,6 @@ #include <stdint.h> #include <string> -#include <boost/function.hpp> - #include <univalue.h> static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; @@ -25,9 +23,9 @@ class CRPCCommand; namespace RPCServer { - void OnStarted(boost::function<void ()> slot); - void OnStopped(boost::function<void ()> slot); - void OnPreCommand(boost::function<void (const CRPCCommand&)> slot); + void OnStarted(std::function<void ()> slot); + void OnStopped(std::function<void ()> slot); + void OnPreCommand(std::function<void (const CRPCCommand&)> slot); } class CBlockIndex; @@ -115,7 +113,7 @@ public: * This is needed to cope with the case in which there is no HTTP server, but * only GUI RPC console, and to break the dependency of pcserver on httprpc. */ - virtual RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) = 0; + virtual RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) = 0; }; /** Set the factory function for timers */ @@ -129,7 +127,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); * Run func nSeconds from now. * Overrides previous timer <name> (if any). */ -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds); +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds); typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); diff --git a/src/scheduler.h b/src/scheduler.h index 613fc1653f..27412a15b4 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -7,8 +7,8 @@ // // NOTE: -// boost::thread / boost::function / boost::chrono should be ported to -// std::thread / std::function / std::chrono when we support C++11. +// boost::thread / boost::chrono should be ported to std::thread / std::chrono +// when we support C++11. // #include <boost/chrono/chrono.hpp> #include <boost/thread.hpp> diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 8ecf0bbdac..f4e5313a78 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -247,10 +247,10 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { static const CScriptNum bnZero(0); static const CScriptNum bnOne(1); - static const CScriptNum bnFalse(0); - static const CScriptNum bnTrue(1); + // static const CScriptNum bnFalse(0); + // static const CScriptNum bnTrue(1); static const valtype vchFalse(0); - static const valtype vchZero(0); + // static const valtype vchZero(0); static const valtype vchTrue(1, 1); CScript::const_iterator pc = script.begin(); diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 6f47b725fb..7bb8d9941b 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -15,28 +15,6 @@ #include <boost/thread.hpp> namespace { - -/** - * We're hashing a nonce into the entries themselves, so we don't need extra - * blinding in the set hash computation. - * - * This may exhibit platform endian dependent behavior but because these are - * nonced hashes (random) and this state is only ever used locally it is safe. - * All that matters is local consistency. - */ -class SignatureCacheHasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin()+4*hash_select, 4); - return u; - } -}; - /** * Valid signature cache, to avoid doing expensive ECDSA signature checking * twice for every transaction (once when accepted into memory pool, and diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 60690583de..55cec4cc8d 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -19,6 +19,27 @@ static const int64_t MAX_MAX_SIG_CACHE_SIZE = 16384; class CPubKey; +/** + * We're hashing a nonce into the entries themselves, so we don't need extra + * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. + */ +class SignatureCacheHasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; + } +}; + class CachingTransactionSignatureChecker : public TransactionSignatureChecker { private: diff --git a/src/serialize.h b/src/serialize.h index e4d72d2348..e82ddf2c5a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -336,11 +336,18 @@ I ReadVarInt(Stream& is) I n = 0; while(true) { unsigned char chData = ser_readdata8(is); + if (n > (std::numeric_limits<I>::max() >> 7)) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n = (n << 7) | (chData & 0x7F); - if (chData & 0x80) + if (chData & 0x80) { + if (n == std::numeric_limits<I>::max()) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n++; - else + } else { return n; + } } } diff --git a/src/streams.h b/src/streams.h index 1387b9cf54..8dc5a19ead 100644 --- a/src/streams.h +++ b/src/streams.h @@ -248,7 +248,8 @@ public: void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -261,7 +262,8 @@ public: void insert(iterator it, const char* first, const char* last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -339,6 +341,8 @@ public: void read(char* pch, size_t nSize) { + if (nSize == 0) return; + // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) diff --git a/src/support/cleanse.h b/src/support/cleanse.h index 3e02aa8fd1..f020216c73 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -8,6 +8,7 @@ #include <stdlib.h> +// Attempt to overwrite data in the specified memory span. void memory_cleanse(void *ptr, size_t len); #endif // BITCOIN_SUPPORT_CLEANSE_H diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 5d1c5b78d1..3812490ec0 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -84,36 +84,43 @@ BOOST_AUTO_TEST_CASE(addrman_simple) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 1: Does Addrman respond correctly when empty. - BOOST_CHECK(addrman.size() == 0); + // Test: Does Addrman respond correctly when empty. + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null = addrman.Select(); - BOOST_CHECK(addr_null.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); - // Test 2: Does Addrman::Add work as expected. + // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret1 = addrman.Select(); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 3: Does IP address deduplication work correctly. + // Test: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1_dup, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(!addrman.Add(CAddress(addr1_dup, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); - // Test 5: New table has one addr and we add a diff addr we should + // Test: New table has one addr and we add a diff addr we should // have two addrs. CService addr2 = ResolveService("250.1.1.2", 8333); - addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 2); + BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 2); - // Test 6: AddrMan::Clear() should empty the new table. + // Test: AddrMan::Clear() should empty the new table. addrman.Clear(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null2 = addrman.Select(); - BOOST_CHECK(addr_null2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null2.ToString(), "[::]:0"); + + // Test: AddrMan::Add multiple addresses works as expected + std::vector<CAddress> vAddr; + vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); + vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); + BOOST_CHECK(addrman.Add(vAddr, source)); + BOOST_CHECK_EQUAL(addrman.size(), 2); } BOOST_AUTO_TEST_CASE(addrman_ports) @@ -125,26 +132,26 @@ BOOST_AUTO_TEST_CASE(addrman_ports) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CService addr1_port = ResolveService("250.1.1.1", 8334); addrman.Add(CAddress(addr1_port, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(); - BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); - // Test 8: Add same IP but diff port to tried table, it doesn't get added. + // Test: Add same IP but diff port to tried table, it doesn't get added. // Perhaps this is not ideal behavior but it is the current behavior. addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret3 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } @@ -157,25 +164,25 @@ BOOST_AUTO_TEST_CASE(addrman_select) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 9: Select from new with 1 addr in new. + // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret1 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 10: move addr to tried, select from new expected nothing returned. + // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); CAddrInfo addr_ret3 = addrman.Select(); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); // Add three addresses to new table. @@ -199,14 +206,15 @@ BOOST_AUTO_TEST_CASE(addrman_select) addrman.Add(CAddress(addr7, NODE_NONE), ResolveService("250.1.1.3", 8333)); addrman.Good(CAddress(addr7, NODE_NONE)); - // Test 11: 6 addrs + 1 addr from last test = 7. - BOOST_CHECK(addrman.size() == 7); + // Test: 6 addrs + 1 addr from last test = 7. + BOOST_CHECK_EQUAL(addrman.size(), 7); - // Test 12: Select pulls from new and tried regardless of port number. - BOOST_CHECK(addrman.Select().ToString() == "250.4.6.6:8333"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.2.2:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.3.3:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.4.4.4:8333"); + // Test: Select pulls from new and tried regardless of port number. + std::set<uint16_t> ports; + for (int i = 0; i < 20; ++i) { + ports.insert(addrman.Select().GetPort()); + } + BOOST_CHECK_EQUAL(ports.size(), 3); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) @@ -218,24 +226,24 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 18; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); - //Test 13: No collision in new table yet. - BOOST_CHECK(addrman.size() == i); + //Test: No collision in new table yet. + BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 14: new table collision! + //Test: new table collision! CService addr1 = ResolveService("250.1.1.18"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 17); + BOOST_CHECK_EQUAL(addrman.size(), 17); CService addr2 = ResolveService("250.1.1.19"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 18); + BOOST_CHECK_EQUAL(addrman.size(), 18); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) @@ -247,25 +255,25 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 80; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); addrman.Good(CAddress(addr, NODE_NONE)); - //Test 15: No collision in tried table yet. + //Test: No collision in tried table yet. BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 16: tried table collision! + //Test: tried table collision! CService addr1 = ResolveService("250.1.1.80"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 79); + BOOST_CHECK_EQUAL(addrman.size(), 79); CService addr2 = ResolveService("250.1.1.81"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 80); + BOOST_CHECK_EQUAL(addrman.size(), 80); } BOOST_AUTO_TEST_CASE(addrman_find) @@ -275,7 +283,7 @@ BOOST_AUTO_TEST_CASE(addrman_find) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -288,23 +296,20 @@ BOOST_AUTO_TEST_CASE(addrman_find) addrman.Add(addr2, source2); addrman.Add(addr3, source1); - // Test 17: ensure Find returns an IP matching what we searched on. + // Test: ensure Find returns an IP matching what we searched on. CAddrInfo* info1 = addrman.Find(addr1); - BOOST_CHECK(info1); - if (info1) - BOOST_CHECK(info1->ToString() == "250.1.2.1:8333"); + BOOST_REQUIRE(info1); + BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); // Test 18; Find does not discriminate by port number. CAddrInfo* info2 = addrman.Find(addr2); - BOOST_CHECK(info2); - if (info2 && info1) - BOOST_CHECK(info2->ToString() == info1->ToString()); + BOOST_REQUIRE(info2); + BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); - // Test 19: Find returns another IP matching what we searched on. + // Test: Find returns another IP matching what we searched on. CAddrInfo* info3 = addrman.Find(addr3); - BOOST_CHECK(info3); - if (info3) - BOOST_CHECK(info3->ToString() == "251.255.2.1:8333"); + BOOST_REQUIRE(info3); + BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) @@ -314,7 +319,7 @@ BOOST_AUTO_TEST_CASE(addrman_create) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -322,11 +327,11 @@ BOOST_AUTO_TEST_CASE(addrman_create) int nId; CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); - // Test 20: The result should be the same as the input addr. - BOOST_CHECK(pinfo->ToString() == "250.1.2.1:8333"); + // Test: The result should be the same as the input addr. + BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); CAddrInfo* info2 = addrman.Find(addr1); - BOOST_CHECK(info2->ToString() == "250.1.2.1:8333"); + BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } @@ -337,7 +342,7 @@ BOOST_AUTO_TEST_CASE(addrman_delete) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -345,10 +350,10 @@ BOOST_AUTO_TEST_CASE(addrman_delete) int nId; addrman.Create(addr1, source1, &nId); - // Test 21: Delete should actually delete the addr. - BOOST_CHECK(addrman.size() == 1); + // Test: Delete should actually delete the addr. + BOOST_CHECK_EQUAL(addrman.size(), 1); addrman.Delete(nId); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == NULL); } @@ -360,11 +365,11 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - // Test 22: Sanity check, GetAddr should never return anything if addrman + // Test: Sanity check, GetAddr should never return anything if addrman // is empty. - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); std::vector<CAddress> vAddr1 = addrman.GetAddr(); - BOOST_CHECK(vAddr1.size() == 0); + BOOST_CHECK_EQUAL(vAddr1.size(), 0); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false @@ -379,7 +384,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CNetAddr source1 = ResolveIP("250.1.2.1"); CNetAddr source2 = ResolveIP("250.2.3.3"); - // Test 23: Ensure GetAddr works with new addresses. + // Test: Ensure GetAddr works with new addresses. addrman.Add(addr1, source1); addrman.Add(addr2, source2); addrman.Add(addr3, source1); @@ -387,21 +392,20 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) addrman.Add(addr5, source1); // GetAddr returns 23% of addresses, 23% of 5 is 1 rounded down. - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 24: Ensure GetAddr works with new and tried addresses. + // Test: Ensure GetAddr works with new and tried addresses. addrman.Good(CAddress(addr1, NODE_NONE)); addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 25: Ensure GetAddr still returns 23% when addrman has many addrs. + // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { int octet1 = i % 256; - int octet2 = (i / 256) % 256; - int octet3 = (i / (256 * 2)) % 256; - std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + "." + boost::to_string(octet3) + ".23"; + int octet2 = i >> 8 % 256; + std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + ".1.23"; CAddress addr = CAddress(ResolveService(strAddr), NODE_NONE); - + // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); addrman.Add(addr, ResolveIP(strAddr)); @@ -411,10 +415,10 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) std::vector<CAddress> vAddr = addrman.GetAddr(); size_t percent23 = (addrman.size() * 23) / 100; - BOOST_CHECK(vAddr.size() == percent23); - BOOST_CHECK(vAddr.size() == 461); + BOOST_CHECK_EQUAL(vAddr.size(), percent23); + BOOST_CHECK_EQUAL(vAddr.size(), 461); // (Addrman.size() < number of addresses added) due to address collisions. - BOOST_CHECK(addrman.size() == 2007); + BOOST_CHECK_EQUAL(addrman.size(), 2006); } @@ -437,13 +441,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetTriedBucket(nKey1) == 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); - // Test 26: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); - // Test 27: Two addresses with same IP but different ports can map to + // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); @@ -458,9 +462,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoi.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 28: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // never get more than 8 buckets - BOOST_CHECK(buckets.size() == 8); + BOOST_CHECK_EQUAL(buckets.size(), 8); buckets.clear(); for (int j = 0; j < 255; j++) { @@ -470,9 +474,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoj.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 29: IP addresses in the different groups should map to more than + // Test: IP addresses in the different groups should map to more than // 8 buckets. - BOOST_CHECK(buckets.size() == 160); + BOOST_CHECK_EQUAL(buckets.size(), 160); } BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) @@ -492,16 +496,18 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetNewBucket(nKey1) == 786); + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); - // Test 30: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); - // Test 31: Ports should not effect bucket placement in the addr + // Test: Ports should not effect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetNewBucket(nKey1) == info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); std::set<int> buckets; for (int i = 0; i < 255; i++) { @@ -511,9 +517,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoi.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 32: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // always map to the same bucket. - BOOST_CHECK(buckets.size() == 1); + BOOST_CHECK_EQUAL(buckets.size(), 1); buckets.clear(); for (int j = 0; j < 4 * 255; j++) { @@ -524,7 +530,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 33: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to no more // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -536,7 +542,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 34: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to more // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index fd6f88b366..952cf901f0 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -3,15 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "amount.h" +#include "policy/feerate.h" #include "test/test_bitcoin.h" #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(amount_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(MoneyRangeTest) +{ + BOOST_CHECK_EQUAL(MoneyRange(CAmount(-1)), false); + BOOST_CHECK_EQUAL(MoneyRange(MAX_MONEY + CAmount(1)), false); + BOOST_CHECK_EQUAL(MoneyRange(CAmount(1)), true); +} + BOOST_AUTO_TEST_CASE(GetFeeTest) { - CFeeRate feeRate; + CFeeRate feeRate, altFeeRate; feeRate = CFeeRate(0); // Must always return 0 @@ -53,6 +61,11 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK_EQUAL(feeRate.GetFee(8), -1); // Special case: returns -1 instead of 0 BOOST_CHECK_EQUAL(feeRate.GetFee(9), -1); + // check alternate constructor + feeRate = CFeeRate(1000); + altFeeRate = CFeeRate(feeRate); + BOOST_CHECK_EQUAL(feeRate.GetFee(100), altFeeRate.GetFee(100)); + // Check full constructor // default value BOOST_CHECK(CFeeRate(CAmount(-1), 1000) == CFeeRate(-1)); @@ -68,4 +81,28 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) CFeeRate(MAX_MONEY, std::numeric_limits<size_t>::max() >> 1).GetFeePerK(); } +BOOST_AUTO_TEST_CASE(BinaryOperatorTest) +{ + CFeeRate a, b; + a = CFeeRate(1); + b = CFeeRate(2); + BOOST_CHECK(a < b); + BOOST_CHECK(b > a); + BOOST_CHECK(a == a); + BOOST_CHECK(a <= b); + BOOST_CHECK(a <= a); + BOOST_CHECK(b >= a); + BOOST_CHECK(b >= b); + // a should be 0.00000002 BTC/kB now + a += a; + BOOST_CHECK(a == b); +} + +BOOST_AUTO_TEST_CASE(ToStringTest) +{ + CFeeRate feeRate; + feeRate = CFeeRate(1); + BOOST_CHECK_EQUAL(feeRate.ToString(), "0.00000001 BTC/kB"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4d17417179..72e562808a 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -3,12 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/aes.h" +#include "crypto/chacha20.h" #include "crypto/ripemd160.h" #include "crypto/sha1.h" #include "crypto/sha256.h" #include "crypto/sha512.h" #include "crypto/hmac_sha256.h" #include "crypto/hmac_sha512.h" +#include "random.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" #include "test/test_random.h" @@ -187,6 +189,19 @@ void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad } } +void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + ChaCha20 rng(key.data(), key.size()); + rng.SetIV(nonce); + rng.Seek(seek); + std::vector<unsigned char> out = ParseHex(hexout); + std::vector<unsigned char> outres; + outres.resize(out.size()); + rng.Output(outres.data(), outres.size()); + BOOST_CHECK(out == outres); +} + std::string LongTestString(void) { std::string ret; for (int i=0; i<200000; i++) { @@ -439,4 +454,57 @@ BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { "b2eb05e2c39be9fcda6c19078c6a9d1b3f461796d6b0d6b2e0c2a72b4d80e644"); } + +BOOST_AUTO_TEST_CASE(chacha20_testvector) +{ + // Test vector from RFC 7539 + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b9334794cb" + "a40c63e34cdea212c4cf07d41b769a6749f3f630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53ac40c5945398b6eda1a" + "832c89c167eacd901d7e2bf363"); + + // Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0, 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b" + "8f41518a11cc387b669b2ee6586"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000001", 0, 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d79" + "2b1c43fea817e9ad275ae546963"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, + "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b52770" + "62eb7a0433e445f41e3"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 1, 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc4" + "97a0b466e7d6bbdb0041b2f586b"); + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3b" + "e59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc1" + "18be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5" + "a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5" + "360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78" + "fab78c9"); +} + +BOOST_AUTO_TEST_CASE(countbits_tests) +{ + FastRandomContext ctx; + for (int i = 0; i <= 64; ++i) { + if (i == 0) { + // Check handling of zero. + BOOST_CHECK_EQUAL(CountBits(0), 0); + } else if (i < 10) { + for (uint64_t j = 1 << (i - 1); (j >> i) == 0; ++j) { + // Exhaustively test up to 10 bits + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } else { + for (int k = 0; k < 1000; k++) { + // Randomly test 1000 samples of each length above 10 bits. + uint64_t j = ((uint64_t)1) << (i - 1) | ctx.randbits(i - 1); + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index ccd7155627..8cae4e66e8 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include "cuckoocache.h" +#include "script/sigcache.h" #include "test/test_bitcoin.h" #include "random.h" #include <thread> @@ -36,20 +37,6 @@ void insecure_GetRandHash(uint256& t) *(ptr++) = insecure_rand.rand32(); } -/** Definition copied from /src/script/sigcache.cpp - */ -class uint256Hasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin() + 4 * hash_select, 4); - return u; - } -}; /* Test that no values not inserted into the cache are read out of it. @@ -59,7 +46,7 @@ public: BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { insecure_rand = FastRandomContext(true); - CuckooCache::cache<uint256, uint256Hasher> cc{}; + CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); uint256 v; @@ -138,7 +125,7 @@ BOOST_AUTO_TEST_CASE(cuckoocache_hit_rate_ok) double HitRateThresh = 0.98; size_t megabytes = 4; for (double load = 0.1; load < 2; load *= 2) { - double hits = test_cache<CuckooCache::cache<uint256, uint256Hasher>>(megabytes, load); + double hits = test_cache<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes, load); BOOST_CHECK(normalize_hit_rate(hits, load) > HitRateThresh); } } @@ -206,7 +193,7 @@ void test_cache_erase(size_t megabytes) BOOST_AUTO_TEST_CASE(cuckoocache_erase_ok) { size_t megabytes = 4; - test_cache_erase<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + test_cache_erase<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } template <typename Cache> @@ -293,7 +280,7 @@ void test_cache_erase_parallel(size_t megabytes) BOOST_AUTO_TEST_CASE(cuckoocache_erase_parallel_ok) { size_t megabytes = 4; - test_cache_erase_parallel<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + test_cache_erase_parallel<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } @@ -389,7 +376,7 @@ void test_cache_generations() } BOOST_AUTO_TEST_CASE(cuckoocache_generations) { - test_cache_generations<CuckooCache::cache<uint256, uint256Hasher>>(); + test_cache_generations<CuckooCache::cache<uint256, SignatureCacheHasher>>(); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 5c054ed3e8..e35a7ce569 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -349,7 +349,7 @@ ["2147483647", "0x04 0xFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["2147483648", "0x05 0x0000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["549755813887", "0x05 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], -["549755813888", "0x06 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], +["549755813888", "0x06 0x000000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["9223372036854775807", "0x08 0xFFFFFFFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["-1", "0x01 0x81 EQUAL", "P2SH,STRICTENC", "OK", "Numbers are little-endian with the MSB being a sign bit"], ["-127", "0x01 0xFF EQUAL", "P2SH,STRICTENC", "OK"], diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index d52104b4cc..656aec606b 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -39,17 +39,18 @@ static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) BOOST_AUTO_TEST_CASE(block_subsidy_test) { - TestBlockSubsidyHalvings(Params(CBaseChainParams::MAIN).GetConsensus()); // As in main + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + TestBlockSubsidyHalvings(chainParams->GetConsensus()); // As in main TestBlockSubsidyHalvings(150); // As in regtest TestBlockSubsidyHalvings(1000); // Just another interval } BOOST_AUTO_TEST_CASE(subsidy_limit_test) { - const Consensus::Params& consensusParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); CAmount nSum = 0; for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) { - CAmount nSubsidy = GetBlockSubsidy(nHeight, consensusParams); + CAmount nSubsidy = GetBlockSubsidy(nHeight, chainParams->GetConsensus()); BOOST_CHECK(nSubsidy <= 50 * COIN); nSum += nSubsidy * 1000; BOOST_CHECK(MoneyRange(nSum)); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index a9d69bb29e..a40060e657 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -195,7 +195,8 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { // Note that by default, these tests run with size accounting enabled. - const CChainParams& chainparams = Params(CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const CChainParams& chainparams = *chainParams; CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; std::unique_ptr<CBlockTemplate> pblocktemplate; CMutableTransaction tx,tx2; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b9ed4952bb..0c7f3e5e23 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -11,6 +11,7 @@ #include "net.h" #include "netbase.h" #include "chainparams.h" +#include "util.h" class CAddrManSerializationMock : public CAddrMan { @@ -72,6 +73,18 @@ CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman) BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(cnode_listen_port) +{ + // test default + unsigned short port = GetListenPort(); + BOOST_CHECK(port == Params().GetDefaultPort()); + // test set port + unsigned short altPort = 12345; + SoftSetArg("-port", std::to_string(altPort)); + port = GetListenPort(); + BOOST_CHECK(port == altPort); +} + BOOST_AUTO_TEST_CASE(caddrdb_read) { CAddrManUncorrupted addrmanUncorrupted; diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index bc2f49ef3f..6bfd315647 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -16,7 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool; + CBlockPolicyEstimator feeEst; + CTxMemPool mpool(&feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); @@ -49,8 +50,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int blocknum = 0; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs @@ -74,20 +75,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, ++blocknum); block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee); } } @@ -99,18 +94,19 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Second highest feerate has 100% chance of being included by 2 blocks, // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { - origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); if (i > 2) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - if (i > 1) { + if (i % 2 == 0) { //At scale 2, test logic is only correct for even targets BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); } - else { - BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) { + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -118,10 +114,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -139,10 +135,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); } - int answerFound; for (int i = 1; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } // Mine all those transactions @@ -155,16 +149,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265); + mpool.removeForBlock(block, 266); block.clear(); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 400 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) { + while (blocknum < 665) { for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; @@ -179,9 +173,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); block.clear(); } - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee @@ -190,8 +184,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 4ca6f1caf0..3b79f8000d 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -16,69 +16,59 @@ BOOST_FIXTURE_TEST_SUITE(pow_tests, BasicTestingSetup) /* Test calculation of next difficulty target with no constraints applying */ BOOST_AUTO_TEST_CASE(get_next_work) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1261130161; // Block #30240 CBlockIndex pindexLast; pindexLast.nHeight = 32255; pindexLast.nTime = 1262152739; // Block #32255 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00d86a); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00d86a); } /* Test the constraint on the upper bound for next work */ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1231006505; // Block #0 CBlockIndex pindexLast; pindexLast.nHeight = 2015; pindexLast.nTime = 1233061996; // Block #2015 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00ffff); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00ffff); } /* Test the constraint on the lower bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1279008237; // Block #66528 CBlockIndex pindexLast; pindexLast.nHeight = 68543; pindexLast.nTime = 1279297671; // Block #68543 pindexLast.nBits = 0x1c05a3f4; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1c0168fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1c0168fd); } /* Test the constraint on the upper bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1263163443; // NOTE: Not an actual block time CBlockIndex pindexLast; pindexLast.nHeight = 46367; pindexLast.nTime = 1269211443; // Block #46367 pindexLast.nBits = 0x1c387f6f; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00e1fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00e1fd); } BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); std::vector<CBlockIndex> blocks(10000); for (int i = 0; i < 10000; i++) { blocks[i].pprev = i ? &blocks[i - 1] : NULL; blocks[i].nHeight = i; - blocks[i].nTime = 1269211443 + i * params.nPowTargetSpacing; + blocks[i].nTime = 1269211443 + i * chainParams->GetConsensus().nPowTargetSpacing; blocks[i].nBits = 0x207fffff; /* target 0x7fffff000... */ blocks[i].nChainWork = i ? blocks[i - 1].nChainWork + GetBlockProof(blocks[i - 1]) : arith_uint256(0); } @@ -88,7 +78,7 @@ BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) CBlockIndex *p2 = &blocks[GetRand(10000)]; CBlockIndex *p3 = &blocks[GetRand(10000)]; - int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, params); + int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, chainParams->GetConsensus()); BOOST_CHECK_EQUAL(tdiff, p1->GetBlockTime() - p2->GetBlockTime()); } } diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index bd8a7819a4..cfed5e347e 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -28,6 +28,7 @@ class prevector_tester { typedef typename pretype::size_type Size; bool passed = true; FastRandomContext rand_cache; + uint256 rand_seed; template <typename A, typename B> @@ -183,13 +184,12 @@ public: } ~prevector_tester() { - BOOST_CHECK_MESSAGE(passed, "insecure_rand_Rz: " - << rand_cache.Rz - << ", insecure_rand_Rw: " - << rand_cache.Rw); + BOOST_CHECK_MESSAGE(passed, "insecure_rand: " + rand_seed.ToString()); } + prevector_tester() { seed_insecure_rand(); + rand_seed = insecure_rand_seed; rand_cache = insecure_rand_ctx; } }; diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index d2c46c0daa..8596734226 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -15,5 +15,39 @@ BOOST_AUTO_TEST_CASE(osrandom_tests) BOOST_CHECK(Random_SanityCheck()); } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_CASE(fastrandom_tests) +{ + // Check that deterministic FastRandomContexts are deterministic + FastRandomContext ctx1(true); + FastRandomContext ctx2(true); + + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + BOOST_CHECK_EQUAL(ctx1.randbits(7), ctx2.randbits(7)); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + + // Check that a nondeterministic ones are not + FastRandomContext ctx3; + FastRandomContext ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal +} +BOOST_AUTO_TEST_CASE(fastrandom_randbits) +{ + FastRandomContext ctx1; + FastRandomContext ctx2; + for (int bits = 0; bits < 63; ++bits) { + for (int j = 0; j < 1000; ++j) { + uint64_t rangebits = ctx1.randbits(bits); + BOOST_CHECK_EQUAL(rangebits >> bits, 0); + uint64_t range = ((uint64_t)1) << bits | rangebits; + uint64_t rand = ctx2.randrange(range); + BOOST_CHECK(rand < range); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index bda3819662..cb625bda11 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -27,7 +27,8 @@ #include <boost/thread.hpp> -FastRandomContext insecure_rand_ctx(true); +uint256 insecure_rand_seed = GetRandHash(); +FastRandomContext insecure_rand_ctx(insecure_rand_seed); extern bool fPrintToConsole; extern void noui_connect(); diff --git a/src/test/test_random.h b/src/test/test_random.h index 4a1637ac72..318c44df4d 100644 --- a/src/test/test_random.h +++ b/src/test/test_random.h @@ -8,11 +8,17 @@ #include "random.h" +extern uint256 insecure_rand_seed; extern FastRandomContext insecure_rand_ctx; static inline void seed_insecure_rand(bool fDeterministic = false) { - insecure_rand_ctx = FastRandomContext(fDeterministic); + if (fDeterministic) { + insecure_rand_seed = uint256(); + } else { + insecure_rand_seed = GetRandHash(); + } + insecure_rand_ctx = FastRandomContext(insecure_rand_seed); } static inline uint32_t insecure_rand(void) diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp new file mode 100644 index 0000000000..b7affaacde --- /dev/null +++ b/src/test/torcontrol_tests.cpp @@ -0,0 +1,199 @@ +// Copyright (c) 2017 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include "test/test_bitcoin.h" +#include "torcontrol.cpp" + +#include <boost/test/unit_test.hpp> + + +BOOST_FIXTURE_TEST_SUITE(torcontrol_tests, BasicTestingSetup) + +void CheckSplitTorReplyLine(std::string input, std::string command, std::string args) +{ + BOOST_TEST_MESSAGE(std::string("CheckSplitTorReplyLine(") + input + ")"); + auto ret = SplitTorReplyLine(input); + BOOST_CHECK_EQUAL(ret.first, command); + BOOST_CHECK_EQUAL(ret.second, args); +} + +BOOST_AUTO_TEST_CASE(util_SplitTorReplyLine) +{ + // Data we should receive during normal usage + CheckSplitTorReplyLine( + "PROTOCOLINFO PIVERSION", + "PROTOCOLINFO", "PIVERSION"); + CheckSplitTorReplyLine( + "AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", + "AUTH", "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\""); + CheckSplitTorReplyLine( + "AUTH METHODS=NULL", + "AUTH", "METHODS=NULL"); + CheckSplitTorReplyLine( + "AUTH METHODS=HASHEDPASSWORD", + "AUTH", "METHODS=HASHEDPASSWORD"); + CheckSplitTorReplyLine( + "VERSION Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", + "VERSION", "Tor=\"0.2.9.8 (git-a0df013ea241b026)\""); + CheckSplitTorReplyLine( + "AUTHCHALLENGE SERVERHASH=aaaa SERVERNONCE=bbbb", + "AUTHCHALLENGE", "SERVERHASH=aaaa SERVERNONCE=bbbb"); + + // Other valid inputs + CheckSplitTorReplyLine("COMMAND", "COMMAND", ""); + CheckSplitTorReplyLine("COMMAND SOME ARGS", "COMMAND", "SOME ARGS"); + + // These inputs are valid because PROTOCOLINFO accepts an OtherLine that is + // just an OptArguments, which enables multiple spaces to be present + // between the command and arguments. + CheckSplitTorReplyLine("COMMAND ARGS", "COMMAND", " ARGS"); + CheckSplitTorReplyLine("COMMAND EVEN+more ARGS", "COMMAND", " EVEN+more ARGS"); +} + +void CheckParseTorReplyMapping(std::string input, std::map<std::string,std::string> expected) +{ + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(") + input + ")"); + auto ret = ParseTorReplyMapping(input); + BOOST_CHECK_EQUAL(ret.size(), expected.size()); + auto r_it = ret.begin(); + auto e_it = expected.begin(); + while (r_it != ret.end() && e_it != expected.end()) { + BOOST_CHECK_EQUAL(r_it->first, e_it->first); + BOOST_CHECK_EQUAL(r_it->second, e_it->second); + r_it++; + e_it++; + } +} + +BOOST_AUTO_TEST_CASE(util_ParseTorReplyMapping) +{ + // Data we should receive during normal usage + CheckParseTorReplyMapping( + "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", { + {"METHODS", "COOKIE,SAFECOOKIE"}, + {"COOKIEFILE", "/home/x/.tor/control_auth_cookie"}, + }); + CheckParseTorReplyMapping( + "METHODS=NULL", { + {"METHODS", "NULL"}, + }); + CheckParseTorReplyMapping( + "METHODS=HASHEDPASSWORD", { + {"METHODS", "HASHEDPASSWORD"}, + }); + CheckParseTorReplyMapping( + "Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", { + {"Tor", "0.2.9.8 (git-a0df013ea241b026)"}, + }); + CheckParseTorReplyMapping( + "SERVERHASH=aaaa SERVERNONCE=bbbb", { + {"SERVERHASH", "aaaa"}, + {"SERVERNONCE", "bbbb"}, + }); + CheckParseTorReplyMapping( + "ServiceID=exampleonion1234", { + {"ServiceID", "exampleonion1234"}, + }); + CheckParseTorReplyMapping( + "PrivateKey=RSA1024:BLOB", { + {"PrivateKey", "RSA1024:BLOB"}, + }); + CheckParseTorReplyMapping( + "ClientAuth=bob:BLOB", { + {"ClientAuth", "bob:BLOB"}, + }); + + // Other valid inputs + CheckParseTorReplyMapping( + "Foo=Bar=Baz Spam=Eggs", { + {"Foo", "Bar=Baz"}, + {"Spam", "Eggs"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar=Baz\"", { + {"Foo", "Bar=Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar Baz\"", { + {"Foo", "Bar Baz"}, + }); + + // Escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\ Baz\"", { + {"Foo", "Bar Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\Baz\"", { + {"Foo", "BarBaz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\@Baz\"", { + {"Foo", "Bar@Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\"Baz\" Spam=\"\\\"Eggs\\\"\"", { + {"Foo", "Bar\"Baz"}, + {"Spam", "\"Eggs\""}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\\Baz\"", { + {"Foo", "Bar\\Baz"}, + }); + + // C escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\nBaz\\t\" Spam=\"\\rEggs\" Octals=\"\\1a\\11\\17\\18\\81\\377\\378\\400\\2222\" Final=Check", { + {"Foo", "Bar\nBaz\t"}, + {"Spam", "\rEggs"}, + {"Octals", "\1a\11\17\1" "881\377\37" "8\40" "0\222" "2"}, + {"Final", "Check"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Escaped=\"Escape\\\\\"", { + {"Valid", "Mapping"}, + {"Escaped", "Escape\\"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Bare=\"Escape\\\"", {}); + CheckParseTorReplyMapping( + "OneOctal=\"OneEnd\\1\" TwoOctal=\"TwoEnd\\11\"", { + {"OneOctal", "OneEnd\1"}, + {"TwoOctal", "TwoEnd\11"}, + }); + + // Special handling for null case + // (needed because string comparison reads the null as end-of-string) + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(Null=\"\\0\")")); + auto ret = ParseTorReplyMapping("Null=\"\\0\""); + BOOST_CHECK_EQUAL(ret.size(), 1); + auto r_it = ret.begin(); + BOOST_CHECK_EQUAL(r_it->first, "Null"); + BOOST_CHECK_EQUAL(r_it->second.size(), 1); + BOOST_CHECK_EQUAL(r_it->second[0], '\0'); + + // A more complex valid grammar. PROTOCOLINFO accepts a VersionLine that + // takes a key=value pair followed by an OptArguments, making this valid. + // Because an OptArguments contains no semantic data, there is no point in + // parsing it. + CheckParseTorReplyMapping( + "SOME=args,here MORE optional=arguments here", { + {"SOME", "args,here"}, + }); + + // Inputs that are effectively invalid under the target grammar. + // PROTOCOLINFO accepts an OtherLine that is just an OptArguments, which + // would make these inputs valid. However, + // - This parser is never used in that situation, because the + // SplitTorReplyLine parser enables OtherLine to be skipped. + // - Even if these were valid, an OptArguments contains no semantic data, + // so there is no point in parsing it. + CheckParseTorReplyMapping("ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("EVEN more=ARGS", {}); + CheckParseTorReplyMapping("EVEN+more ARGS", {}); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 5da4907f06..10330c0c23 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -17,8 +17,6 @@ #include <boost/test/unit_test.hpp> -extern std::map<std::string, std::string> mapArgs; - BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_criticalsection) @@ -100,52 +98,67 @@ BOOST_AUTO_TEST_CASE(util_DateTimeStrFormat) BOOST_CHECK_EQUAL(DateTimeStrFormat("%a, %d %b %Y %H:%M:%S +0000", 1317425777), "Fri, 30 Sep 2011 23:36:17 +0000"); } +class TestArgsManager : public ArgsManager +{ +public: + std::map<std::string, std::string>& GetMapArgs() + { + return mapArgs; + }; + const std::map<std::string, std::vector<std::string> >& GetMapMultiArgs() + { + return mapMultiArgs; + }; +}; + BOOST_AUTO_TEST_CASE(util_ParseParameters) { + TestArgsManager testArgs; const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; - ParseParameters(0, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(0, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(1, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(1, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(5, (char**)argv_test); + testArgs.ParseParameters(5, (char**)argv_test); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(mapArgs.size() == 3 && mapMultiArgs.size() == 3); - BOOST_CHECK(IsArgSet("-a") && IsArgSet("-b") && IsArgSet("-ccc") - && !IsArgSet("f") && !IsArgSet("-d")); - BOOST_CHECK(mapMultiArgs.count("-a") && mapMultiArgs.count("-b") && mapMultiArgs.count("-ccc") - && !mapMultiArgs.count("f") && !mapMultiArgs.count("-d")); - - BOOST_CHECK(mapArgs["-a"] == "" && mapArgs["-ccc"] == "multiple"); - BOOST_CHECK(mapMultiArgs.at("-ccc").size() == 2); + BOOST_CHECK(testArgs.GetMapArgs().size() == 3 && testArgs.GetMapMultiArgs().size() == 3); + BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") + && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); + BOOST_CHECK(testArgs.GetMapMultiArgs().count("-a") && testArgs.GetMapMultiArgs().count("-b") && testArgs.GetMapMultiArgs().count("-ccc") + && !testArgs.GetMapMultiArgs().count("f") && !testArgs.GetMapMultiArgs().count("-d")); + + BOOST_CHECK(testArgs.GetMapArgs()["-a"] == "" && testArgs.GetMapArgs()["-ccc"] == "multiple"); + BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } BOOST_AUTO_TEST_CASE(util_GetArg) { - mapArgs.clear(); - mapArgs["strtest1"] = "string..."; + TestArgsManager testArgs; + testArgs.GetMapArgs().clear(); + testArgs.GetMapArgs()["strtest1"] = "string..."; // strtest2 undefined on purpose - mapArgs["inttest1"] = "12345"; - mapArgs["inttest2"] = "81985529216486895"; + testArgs.GetMapArgs()["inttest1"] = "12345"; + testArgs.GetMapArgs()["inttest2"] = "81985529216486895"; // inttest3 undefined on purpose - mapArgs["booltest1"] = ""; + testArgs.GetMapArgs()["booltest1"] = ""; // booltest2 undefined on purpose - mapArgs["booltest3"] = "0"; - mapArgs["booltest4"] = "1"; - - BOOST_CHECK_EQUAL(GetArg("strtest1", "default"), "string..."); - BOOST_CHECK_EQUAL(GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(GetArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(GetArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(GetArg("inttest3", -1), -1); - BOOST_CHECK_EQUAL(GetBoolArg("booltest1", false), true); - BOOST_CHECK_EQUAL(GetBoolArg("booltest2", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest3", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest4", false), true); + testArgs.GetMapArgs()["booltest3"] = "0"; + testArgs.GetMapArgs()["booltest4"] = "1"; + + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest4", false), true); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index e2b5573abd..79405ec4d1 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -209,7 +209,8 @@ BOOST_AUTO_TEST_CASE(versionbits_test) } // Sanity checks of version bit deployments - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); for (int i=0; i<(int) Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { uint32_t bitmask = VersionBitsMask(mainnetParams, (Consensus::DeploymentPos)i); // Make sure that no deployment tries to set an invalid bit. @@ -235,7 +236,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) { // Check that ComputeBlockVersion will set the appropriate bit correctly // on mainnet. - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); // Use the TESTDUMMY deployment for testing purposes. int64_t bit = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index c1bd95b00f..8a37139f1d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +15,6 @@ #include <set> #include <stdlib.h> -#include <boost/function.hpp> #include <boost/bind.hpp> #include <boost/signals2/signal.hpp> #include <boost/foreach.hpp> @@ -73,8 +73,8 @@ public: class TorControlConnection { public: - typedef boost::function<void(TorControlConnection&)> ConnectionCB; - typedef boost::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; + typedef std::function<void(TorControlConnection&)> ConnectionCB; + typedef std::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; /** Create a new TorControlConnection. */ @@ -105,9 +105,9 @@ public: boost::signals2::signal<void(TorControlConnection &,const TorControlReply &)> async_handler; private: /** Callback when ready for use */ - boost::function<void(TorControlConnection&)> connected; + std::function<void(TorControlConnection&)> connected; /** Callback when connection lost */ - boost::function<void(TorControlConnection&)> disconnected; + std::function<void(TorControlConnection&)> disconnected; /** Libevent event base */ struct event_base *base; /** Connection to control socket */ @@ -250,6 +250,8 @@ bool TorControlConnection::Command(const std::string &cmd, const ReplyHandlerCB& /* Split reply line in the form 'AUTH METHODS=...' into a type * 'AUTH' and arguments 'METHODS=...'. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21) and AUTHCHALLENGE (S3.24). */ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s) { @@ -265,6 +267,10 @@ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s } /** Parse reply arguments in the form 'METHODS=COOKIE,SAFECOOKIE COOKIEFILE=".../control_auth_cookie"'. + * Returns a map of keys to values, or an empty map if there was an error. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21), AUTHCHALLENGE (S3.24), + * and ADD_ONION (S3.27). See also sections 2.1 and 2.3. */ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) { @@ -272,28 +278,74 @@ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string size_t ptr=0; while (ptr < s.size()) { std::string key, value; - while (ptr < s.size() && s[ptr] != '=') { + while (ptr < s.size() && s[ptr] != '=' && s[ptr] != ' ') { key.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); + if (s[ptr] == ' ') // The remaining string is an OptArguments + break; ++ptr; // skip '=' if (ptr < s.size() && s[ptr] == '"') { // Quoted string - ++ptr; // skip '=' + ++ptr; // skip opening '"' bool escape_next = false; - while (ptr < s.size() && (!escape_next && s[ptr] != '"')) { - escape_next = (s[ptr] == '\\'); + while (ptr < s.size() && (escape_next || s[ptr] != '"')) { + // Repeated backslashes must be interpreted as pairs + escape_next = (s[ptr] == '\\' && !escape_next); value.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); ++ptr; // skip closing '"' - /* TODO: unescape value - according to the spec this depends on the - * context, some strings use C-LogPrintf style escape codes, some - * don't. So may be better handled at the call site. + /** + * Unescape value. Per https://spec.torproject.org/control-spec section 2.1.1: + * + * For future-proofing, controller implementors MAY use the following + * rules to be compatible with buggy Tor implementations and with + * future ones that implement the spec as intended: + * + * Read \n \t \r and \0 ... \377 as C escapes. + * Treat a backslash followed by any other character as that character. */ + std::string escaped_value; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '\\') { + // This will always be valid, because if the QuotedString + // ended in an odd number of backslashes, then the parser + // would already have returned above, due to a missing + // terminating double-quote. + ++i; + if (value[i] == 'n') { + escaped_value.push_back('\n'); + } else if (value[i] == 't') { + escaped_value.push_back('\t'); + } else if (value[i] == 'r') { + escaped_value.push_back('\r'); + } else if ('0' <= value[i] && value[i] <= '7') { + size_t j; + // Octal escape sequences have a limit of three octal digits, + // but terminate at the first character that is not a valid + // octal digit if encountered sooner. + for (j = 1; j < 3 && (i+j) < value.size() && '0' <= value[i+j] && value[i+j] <= '7'; ++j) {} + // Tor restricts first digit to 0-3 for three-digit octals. + // A leading digit of 4-7 would therefore be interpreted as + // a two-digit octal. + if (j == 3 && value[i] > '3') { + j--; + } + escaped_value.push_back(strtol(value.substr(i, j).c_str(), NULL, 8)); + // Account for automatic incrementing at loop end + i += j - 1; + } else { + escaped_value.push_back(value[i]); + } + } else { + escaped_value.push_back(value[i]); + } + } + value = escaped_value; } else { // Unquoted value. Note that values can contain '=' at will, just no spaces while (ptr < s.size() && s[ptr] != ' ') { value.push_back(s[ptr]); @@ -323,6 +375,10 @@ static std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size char buffer[128]; size_t n; while ((n=fread(buffer, 1, sizeof(buffer), f)) > 0) { + // Check for reading errors so we don't return any data if we couldn't + // read the entire file (or up to maxsize) + if (ferror(f)) + return std::make_pair(false,""); retval.append(buffer, buffer+n); if (retval.size() > maxsize) break; @@ -439,6 +495,13 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe if ((i = m.find("PrivateKey")) != m.end()) private_key = i->second; } + if (service_id.empty()) { + LogPrintf("tor: Error parsing ADD_ONION parameters:\n"); + for (const std::string &s : reply.lines) { + LogPrintf(" %s\n", SanitizeString(s)); + } + return; + } service = LookupNumeric(std::string(service_id+".onion").c_str(), GetListenPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { @@ -516,6 +579,10 @@ void TorController::authchallenge_cb(TorControlConnection& _conn, const TorContr std::pair<std::string,std::string> l = SplitTorReplyLine(reply.lines[0]); if (l.first == "AUTHCHALLENGE") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); + if (m.empty()) { + LogPrintf("tor: Error parsing AUTHCHALLENGE parameters: %s\n", SanitizeString(l.second)); + return; + } std::vector<uint8_t> serverHash = ParseHex(m["SERVERHASH"]); std::vector<uint8_t> serverNonce = ParseHex(m["SERVERNONCE"]); LogPrint(BCLog::TOR, "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); diff --git a/src/txdb.cpp b/src/txdb.cpp index a3889fdf79..42dc31760b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -169,7 +169,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } -bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex) +bool CBlockTreeDB::LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex) { std::unique_ptr<CDBIterator> pcursor(NewIterator()); diff --git a/src/txdb.h b/src/txdb.h index d9214ba618..117e7201fb 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -15,8 +15,6 @@ #include <utility> #include <vector> -#include <boost/function.hpp> - class CBlockIndex; class CCoinsViewDBCursor; class uint256; @@ -122,7 +120,7 @@ public: bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &list); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex); + bool LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index beb2e0482c..33df0536d0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,7 +5,6 @@ #include "txmempool.h" -#include "clientversion.h" #include "consensus/consensus.h" #include "consensus/tx_verify.h" #include "consensus/validation.h" @@ -17,7 +16,6 @@ #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, @@ -334,8 +332,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool() : - nTransactionsUpdated(0) +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -343,13 +341,6 @@ CTxMemPool::CTxMemPool() : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(); -} - -CTxMemPool::~CTxMemPool() -{ - delete minerPolicyEstimator; } void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) @@ -428,7 +419,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -458,7 +449,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -592,7 +583,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -851,51 +842,6 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -CFeeRate CTxMemPool::estimateFee(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateFee(nBlocks); -} -CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); -} - -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const -{ - try { - LOCK(cs); - fileout << 139900; // version required to read: 0.13.99 or later - fileout << CLIENT_VERSION; // version that wrote the file - minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) -{ - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { @@ -920,6 +866,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD BOOST_FOREACH(txiter descendantIt, setDescendants) { mapTx.modify(descendantIt, update_ancestor_state(0, nFeeDelta, 0, 0)); } + ++nTransactionsUpdated; } } LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); diff --git a/src/txmempool.h b/src/txmempool.h index 4222789510..a91eb5be54 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,6 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "policy/feerate.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -59,11 +60,6 @@ class CTxMemPool; * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * - * If updating the descendant state is skipped, we can mark the entry as - * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal nTxSize/ - * nFee+feeDelta. (This can potentially happen during a reorg, where we limit the - * amount of work we're willing to do to avoid consuming too much CPU.) - * */ class CTxMemPoolEntry @@ -82,9 +78,7 @@ private: // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these - // descendants as well. if nCountWithDescendants is 0, treat this entry as - // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be - // correct. + // descendants as well. uint64_t nCountWithDescendants; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) @@ -115,7 +109,7 @@ public: size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } - // Adjusts the descendant state, if this entry is not dirty. + // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); @@ -398,20 +392,12 @@ enum class MemPoolRemovalReason { * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * - * Adding transactions from a disconnected block can be very time consuming, - * because we don't have a way to limit the number of in-mempool descendants. - * To bound CPU processing, we limit the amount of work we're willing to do - * to properly update the descendant information for a tx being added from - * a disconnected block. If we would exceed the limit, then we instead mark - * the entry as "dirty", and set the feerate for sorting purposes to be equal - * the feerate of the transaction without any descendants. - * */ class CTxMemPool { private: uint32_t nCheckFrequency; //!< Value n means that n times in 2^32 we check. - unsigned int nTransactionsUpdated; + unsigned int nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. @@ -496,8 +482,7 @@ public: /** Create a new CTxMemPool. */ - CTxMemPool(); - ~CTxMemPool(); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -618,19 +603,6 @@ public: TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; - /** Estimate fee rate needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate fee rate needed to get into the next nBlocks */ - CFeeRate estimateFee(int nBlocks) const; - - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile& fileout) const; - bool ReadFeeEstimates(CAutoFile& filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal<void (CTransactionRef)> NotifyEntryAdded; diff --git a/src/util.cpp b/src/util.cpp index 5473799289..653a4f072a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -13,7 +13,6 @@ #include "fs.h" #include "random.h" #include "serialize.h" -#include "sync.h" #include "utilstrencodings.h" #include "utiltime.h" @@ -88,27 +87,11 @@ #include <openssl/rand.h> #include <openssl/conf.h> -// Work around clang compilation problem in Boost 1.46: -// /usr/include/boost/program_options/detail/config_file.hpp:163:17: error: call to function 'to_internal' that is neither visible in the template definition nor found by argument-dependent lookup -// See also: http://stackoverflow.com/questions/10020179/compilation-fail-in-boost-librairies-program-options -// http://clang.debian.net/status.php?version=3.0&key=CANNOT_FIND_FUNCTION -namespace boost { - - namespace program_options { - std::string to_internal(const std::string&); - } - -} // namespace boost - - const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; -CCriticalSection cs_args; -std::map<std::string, std::string> mapArgs; -static std::map<std::string, std::vector<std::string> > _mapMultiArgs; -const std::map<std::string, std::vector<std::string> >& mapMultiArgs = _mapMultiArgs; +ArgsManager gArgs; bool fPrintToConsole = false; bool fPrintToDebugLog = true; @@ -118,7 +101,7 @@ bool fLogIPs = DEFAULT_LOGIPS; std::atomic<bool> fReopenDebugLog(false); CTranslationInterface translationInterface; -/** Log categories bitfield. Leveldb/libevent need special handling if their flags are changed at runtime. */ +/** Log categories bitfield. */ std::atomic<uint32_t> logCategories(0); /** Init OpenSSL library multithreading support */ @@ -295,6 +278,21 @@ std::string ListLogCategories() return ret; } +std::vector<CLogCategoryActive> ListActiveLogCategories() +{ + std::vector<CLogCategoryActive> ret; + for (unsigned int i = 0; i < ARRAYLEN(LogCategories); i++) { + // Omit the special cases. + if (LogCategories[i].flag != BCLog::NONE && LogCategories[i].flag != BCLog::ALL) { + CLogCategoryActive catActive; + catActive.category = LogCategories[i].category; + catActive.active = LogAcceptCategory(LogCategories[i].flag); + ret.push_back(catActive); + } + } + return ret; +} + /** * fStartedNewLine is a state variable held by the calling context that will * suppress printing of the timestamp when multiple calls are made that don't @@ -308,10 +306,14 @@ static std::string LogTimestampStr(const std::string &str, std::atomic_bool *fSt return str; if (*fStartedNewLine) { - int64_t nTimeMicros = GetLogTimeMicros(); + int64_t nTimeMicros = GetTimeMicros(); strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros/1000000); if (fLogTimeMicros) strStamped += strprintf(".%06d", nTimeMicros%1000000); + int64_t mocktime = GetMockTime(); + if (mocktime) { + strStamped += " (mocktime: " + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", mocktime) + ")"; + } strStamped += ' ' + str; } else strStamped = str; @@ -382,11 +384,11 @@ static void InterpretNegativeSetting(std::string& strKey, std::string& strValue) } } -void ParseParameters(int argc, const char* const argv[]) +void ArgsManager::ParseParameters(int argc, const char* const argv[]) { LOCK(cs_args); mapArgs.clear(); - _mapMultiArgs.clear(); + mapMultiArgs.clear(); for (int i = 1; i < argc; i++) { @@ -414,17 +416,23 @@ void ParseParameters(int argc, const char* const argv[]) InterpretNegativeSetting(str, strValue); mapArgs[str] = strValue; - _mapMultiArgs[str].push_back(strValue); + mapMultiArgs[str].push_back(strValue); } } -bool IsArgSet(const std::string& strArg) +std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) +{ + LOCK(cs_args); + return mapMultiArgs.at(strArg); +} + +bool ArgsManager::IsArgSet(const std::string& strArg) { LOCK(cs_args); return mapArgs.count(strArg); } -std::string GetArg(const std::string& strArg, const std::string& strDefault) +std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -432,7 +440,7 @@ std::string GetArg(const std::string& strArg, const std::string& strDefault) return strDefault; } -int64_t GetArg(const std::string& strArg, int64_t nDefault) +int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -440,7 +448,7 @@ int64_t GetArg(const std::string& strArg, int64_t nDefault) return nDefault; } -bool GetBoolArg(const std::string& strArg, bool fDefault) +bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -448,16 +456,16 @@ bool GetBoolArg(const std::string& strArg, bool fDefault) return fDefault; } -bool SoftSetArg(const std::string& strArg, const std::string& strValue) +bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); if (mapArgs.count(strArg)) return false; - mapArgs[strArg] = strValue; + ForceSetArg(strArg, strValue); return true; } -bool SoftSetBoolArg(const std::string& strArg, bool fValue) +bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) { if (fValue) return SoftSetArg(strArg, std::string("1")); @@ -465,10 +473,11 @@ bool SoftSetBoolArg(const std::string& strArg, bool fValue) return SoftSetArg(strArg, std::string("0")); } -void ForceSetArg(const std::string& strArg, const std::string& strValue) +void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); mapArgs[strArg] = strValue; + mapMultiArgs[strArg].push_back(strValue); } @@ -587,7 +596,7 @@ fs::path GetConfigFile(const std::string& confPath) return pathConfigFile; } -void ReadConfigFile(const std::string& confPath) +void ArgsManager::ReadConfigFile(const std::string& confPath) { fs::ifstream streamConfig(GetConfigFile(confPath)); if (!streamConfig.good()) @@ -606,7 +615,7 @@ void ReadConfigFile(const std::string& confPath) InterpretNegativeSetting(strKey, strValue); if (mapArgs.count(strKey) == 0) mapArgs[strKey] = strValue; - _mapMultiArgs[strKey].push_back(strValue); + mapMultiArgs[strKey].push_back(strValue); } } // If datadir is changed in .conf file: diff --git a/src/util.h b/src/util.h index 7998449fee..229478d835 100644 --- a/src/util.h +++ b/src/util.h @@ -16,6 +16,7 @@ #include "compat.h" #include "fs.h" +#include "sync.h" #include "tinyformat.h" #include "utiltime.h" @@ -41,7 +42,6 @@ public: boost::signals2::signal<std::string (const char* psz)> Translate; }; -extern const std::map<std::string, std::vector<std::string> >& mapMultiArgs; extern bool fPrintToConsole; extern bool fPrintToDebugLog; @@ -69,6 +69,12 @@ inline std::string _(const char* psz) void SetupEnvironment(); bool SetupNetworking(); +struct CLogCategoryActive +{ + std::string category; + bool active; +}; + namespace BCLog { enum LogFlags : uint32_t { NONE = 0, @@ -102,9 +108,12 @@ static inline bool LogAcceptCategory(uint32_t category) return (logCategories.load(std::memory_order_relaxed) & category) != 0; } -/** Returns a string with the supported log categories */ +/** Returns a string with the log categories. */ std::string ListLogCategories(); +/** Returns a vector of the active log categories. */ +std::vector<CLogCategoryActive> ListActiveLogCategories(); + /** Return true if str parses as a log category and set the flags in f */ bool GetLogCategory(uint32_t *f, const std::string *str); @@ -139,7 +148,6 @@ bool error(const char* fmt, const Args&... args) } void PrintExceptionContinue(const std::exception *pex, const char* pszThread); -void ParseParameters(int argc, const char*const argv[]); void FileCommit(FILE *file); bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); @@ -154,7 +162,6 @@ fs::path GetConfigFile(const std::string& confPath); fs::path GetPidFile(); void CreatePidFile(const fs::path &path, pid_t pid); #endif -void ReadConfigFile(const std::string& confPath); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif @@ -171,6 +178,16 @@ inline bool IsSwitchChar(char c) #endif } +class ArgsManager +{ +protected: + CCriticalSection cs_args; + std::map<std::string, std::string> mapArgs; + std::map<std::string, std::vector<std::string> > mapMultiArgs; +public: + void ParseParameters(int argc, const char*const argv[]); + void ReadConfigFile(const std::string& confPath); + std::vector<std::string> GetArgs(const std::string& strArg); /** * Return true if the given argument has been manually set * @@ -226,6 +243,55 @@ bool SoftSetBoolArg(const std::string& strArg, bool fValue); // Forces a arg setting, used only in testing void ForceSetArg(const std::string& strArg, const std::string& strValue); +}; + +extern ArgsManager gArgs; + +// wrappers using the global ArgsManager: +static inline void ParseParameters(int argc, const char*const argv[]) +{ + gArgs.ParseParameters(argc, argv); +} + +static inline void ReadConfigFile(const std::string& confPath) +{ + gArgs.ReadConfigFile(confPath); +} + +static inline bool SoftSetArg(const std::string& strArg, const std::string& strValue) +{ + return gArgs.SoftSetArg(strArg, strValue); +} + +static inline void ForceSetArg(const std::string& strArg, const std::string& strValue) +{ + gArgs.ForceSetArg(strArg, strValue); +} + +static inline bool IsArgSet(const std::string& strArg) +{ + return gArgs.IsArgSet(strArg); +} + +static inline std::string GetArg(const std::string& strArg, const std::string& strDefault) +{ + return gArgs.GetArg(strArg, strDefault); +} + +static inline int64_t GetArg(const std::string& strArg, int64_t nDefault) +{ + return gArgs.GetArg(strArg, nDefault); +} + +static inline bool GetBoolArg(const std::string& strArg, bool fDefault) +{ + return gArgs.GetBoolArg(strArg, fDefault); +} + +static inline bool SoftSetBoolArg(const std::string& strArg, bool fValue) +{ + return gArgs.SoftSetBoolArg(strArg, fValue); +} /** * Format a string to be used as group of options in help messages diff --git a/src/utiltime.cpp b/src/utiltime.cpp index a9936a645a..e07069125d 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -9,14 +9,17 @@ #include "utiltime.h" +#include <atomic> + #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread.hpp> -static int64_t nMockTime = 0; //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For unit testing int64_t GetTime() { - if (nMockTime) return nMockTime; + int64_t mocktime = nMockTime.load(std::memory_order_relaxed); + if (mocktime) return mocktime; time_t now = time(NULL); assert(now > 0); @@ -25,7 +28,12 @@ int64_t GetTime() void SetMockTime(int64_t nMockTimeIn) { - nMockTime = nMockTimeIn; + nMockTime.store(nMockTimeIn, std::memory_order_relaxed); +} + +int64_t GetMockTime() +{ + return nMockTime.load(std::memory_order_relaxed); } int64_t GetTimeMillis() @@ -49,14 +57,6 @@ int64_t GetSystemTimeInSeconds() return GetTimeMicros()/1000000; } -/** Return a time useful for the debug log */ -int64_t GetLogTimeMicros() -{ - if (nMockTime) return nMockTime*1000000; - - return GetTimeMicros(); -} - void MilliSleep(int64_t n) { diff --git a/src/utiltime.h b/src/utiltime.h index cc3290c631..8ae8540b89 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -23,8 +23,8 @@ int64_t GetTime(); int64_t GetTimeMillis(); int64_t GetTimeMicros(); int64_t GetSystemTimeInSeconds(); // Like GetTime(), but not mockable -int64_t GetLogTimeMicros(); void SetMockTime(int64_t nMockTimeIn); +int64_t GetMockTime(); void MilliSleep(int64_t n); std::string DateTimeStrFormat(const char* pszFormat, int64_t nTime); diff --git a/src/validation.cpp b/src/validation.cpp index c9dda96e88..73466b9df7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -6,6 +6,7 @@ #include "validation.h" #include "arith_uint256.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -81,7 +82,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool; +CBlockPolicyEstimator feeEstimator; +CTxMemPool mempool(&feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -155,39 +157,6 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -/* Use this class to start tracking transactions that are removed from the - * mempool and pass all those transactions through SyncTransaction when the - * object goes out of scope. This is currently only used to call SyncTransaction - * on conflicts removed from the mempool during block connection. Applied in - * ActivateBestChain around ActivateBestStep which in turn calls: - * ConnectTip->removeForBlock->removeConflicts - */ -class MemPoolConflictRemovalTracker -{ -private: - std::vector<CTransactionRef> conflictedTxs; - CTxMemPool &pool; - -public: - MemPoolConflictRemovalTracker(CTxMemPool &_pool) : pool(_pool) { - pool.NotifyEntryRemoved.connect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - } - - void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { - if (reason == MemPoolRemovalReason::CONFLICT) { - conflictedTxs.push_back(txRemoved); - } - } - - ~MemPoolConflictRemovalTracker() { - pool.NotifyEntryRemoved.disconnect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - for (const auto& tx : conflictedTxs) { - GetMainSignals().SyncTransaction(*tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } - conflictedTxs.clear(); - } -}; - CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { // Find the first block the caller has in the main chain @@ -786,7 +755,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - GetMainSignals().SyncTransaction(tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + GetMainSignals().TransactionAddedToMempool(ptx); return true; } @@ -976,6 +945,7 @@ bool IsInitialBlockDownload() return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; + LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); latchToFalse.store(true, std::memory_order_relaxed); return false; } @@ -1134,10 +1104,7 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; - if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error)) { - return false; - } - return true; + return VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error); } int GetSpendHeight(const CCoinsViewCache& inputs) @@ -1315,24 +1282,36 @@ bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint return fClean; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +enum DisconnectResult { - assert(pindex->GetBlockHash() == view.GetBestBlock()); + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; - if (pfClean) - *pfClean = false; +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */ +static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) +{ + assert(pindex->GetBlockHash() == view.GetBestBlock()); bool fClean = true; CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) - return error("DisconnectBlock(): no undo data available"); - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock(): failure reading undo data"); + if (pos.IsNull()) { + error("DisconnectBlock(): no undo data available"); + return DISCONNECT_FAILED; + } + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } - if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock(): block and undo data inconsistent"); + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1361,8 +1340,10 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // restore inputs if (i > 0) { // not coinbases const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; const CTxInUndo &undo = txundo.vprevout[j]; @@ -1375,12 +1356,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); - if (pfClean) { - *pfClean = fClean; - return true; - } - - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } void static FlushBlockFile(bool fFinalize = false) @@ -1468,8 +1444,11 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) { AssertLockHeld(cs_main); assert(pindex); @@ -1672,7 +1651,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) - return state.DoS(100, false); + return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); @@ -1708,12 +1687,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); - // Watch for changes to the previous coinbase transaction. - static uint256 hashPrevBestCoinBase; - GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0]->GetHash(); - - int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); @@ -1911,14 +1884,15 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); // Read block from disk. - CBlock block; + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + CBlock& block = *pblock; if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); @@ -1953,9 +1927,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - for (const auto& tx : block.vtx) { - GetMainSignals().SyncTransaction(*tx, pindexDelete->pprev, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } + GetMainSignals().BlockDisconnected(pblock); return true; } @@ -1965,36 +1937,92 @@ static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; +struct PerBlockConnectTrace { + CBlockIndex* pindex = NULL; + std::shared_ptr<const CBlock> pblock; + std::shared_ptr<std::vector<CTransactionRef>> conflictedTxs; + PerBlockConnectTrace() : conflictedTxs(std::make_shared<std::vector<CTransactionRef>>()) {} +}; /** * Used to track blocks whose transactions were applied to the UTXO state as a * part of a single ActivateBestChainStep call. + * + * This class also tracks transactions that are removed from the mempool as + * conflicts (per block) and can be used to pass all those transactions + * through SyncTransaction. + * + * This class assumes (and asserts) that the conflicted transactions for a given + * block are added via mempool callbacks prior to the BlockConnected() associated + * with those transactions. If any transactions are marked conflicted, it is + * assumed that an associated block will always be added. + * + * This class is single-use, once you call GetBlocksConnected() you have to throw + * it away and make a new one. */ -struct ConnectTrace { - std::vector<std::pair<CBlockIndex*, std::shared_ptr<const CBlock> > > blocksConnected; +class ConnectTrace { +private: + std::vector<PerBlockConnectTrace> blocksConnected; + CTxMemPool &pool; + +public: + ConnectTrace(CTxMemPool &_pool) : blocksConnected(1), pool(_pool) { + pool.NotifyEntryRemoved.connect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); + } + + ~ConnectTrace() { + pool.NotifyEntryRemoved.disconnect(boost::bind(&ConnectTrace::NotifyEntryRemoved, this, _1, _2)); + } + + void BlockConnected(CBlockIndex* pindex, std::shared_ptr<const CBlock> pblock) { + assert(!blocksConnected.back().pindex); + assert(pindex); + assert(pblock); + blocksConnected.back().pindex = pindex; + blocksConnected.back().pblock = std::move(pblock); + blocksConnected.emplace_back(); + } + + std::vector<PerBlockConnectTrace>& GetBlocksConnected() { + // We always keep one extra block at the end of our list because + // blocks are added after all the conflicted transactions have + // been filled in. Thus, the last entry should always be an empty + // one waiting for the transactions from the next block. We pop + // the last entry here to make sure the list we return is sane. + assert(!blocksConnected.back().pindex); + assert(blocksConnected.back().conflictedTxs->empty()); + blocksConnected.pop_back(); + return blocksConnected; + } + + void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { + assert(!blocksConnected.back().pindex); + if (reason == MemPoolRemovalReason::CONFLICT) { + blocksConnected.back().conflictedTxs->emplace_back(std::move(txRemoved)); + } + } }; /** * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. * - * The block is always added to connectTrace (either after loading from disk or by copying - * pblock) - if that is not intended, care must be taken to remove the last entry in - * blocksConnected in case of failure. + * The block is added to connectTrace if connection succeeds. */ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); + std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - connectTrace.blocksConnected.emplace_back(pindexNew, pblockNew); if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); + pthisBlock = pblockNew; } else { - connectTrace.blocksConnected.emplace_back(pindexNew, pblock); + pthisBlock = pblock; } - const CBlock& blockConnecting = *connectTrace.blocksConnected.back().second; + const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; @@ -2028,6 +2056,8 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + + connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } @@ -2146,8 +2176,6 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c state = CValidationState(); fInvalidFound = true; fContinue = false; - // If we didn't actually connect the block, don't notify listeners about it - connectTrace.blocksConnected.pop_back(); break; } else { // A system error occurred (disk space, database error, ...). @@ -2219,18 +2247,11 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, break; const CBlockIndex *pindexFork; - ConnectTrace connectTrace; bool fInitialDownload; { LOCK(cs_main); - { // TODO: Temporarily ensure that mempool removals are notified before - // connected transactions. This shouldn't matter, but the abandoned - // state of transactions in our wallet is currently cleared when we - // receive another notification and there is a race condition where - // notification of a connected conflict might cause an outside process - // to abandon a transaction and then have it inadvertently cleared by - // the notification that the conflicted transaction was evicted. - MemPoolConflictRemovalTracker mrt(mempool); + ConnectTrace connectTrace(mempool); // Destructed before cs_main is unlocked + CBlockIndex *pindexOldTip = chainActive.Tip(); if (pindexMostWork == NULL) { pindexMostWork = FindMostWorkChain(); @@ -2253,16 +2274,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); - // throw all transactions though the signal-interface - - } // MemPoolConflictRemovalTracker destroyed and conflict evictions are notified - - // Transactions in the connected block are notified - for (const auto& pair : connectTrace.blocksConnected) { - assert(pair.second); - const CBlock& block = *(pair.second); - for (unsigned int i = 0; i < block.vtx.size(); i++) - GetMainSignals().SyncTransaction(*block.vtx[i], pair.first, i); + for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { + assert(trace.pblock && trace.pindex); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2284,6 +2298,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return false; } + int nStopAtHeight = GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + return true; } @@ -2426,7 +2443,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2434,7 +2451,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -2640,10 +2657,12 @@ static bool CheckIndexAgainstCheckpoint(const CBlockIndex* pindexPrev, CValidati return true; int nHeight = pindexPrev->nHeight+1; - // Don't accept any forks from the main chain prior to last checkpoint + // Don't accept any forks from the main chain prior to last checkpoint. + // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our + // MapBlockIndex. CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(chainparams.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight)); + return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); return true; } @@ -2844,7 +2863,7 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state CBlockIndex* pindexPrev = NULL; BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) - return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); + return state.DoS(10, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); @@ -2954,7 +2973,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (dbp == NULL) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) AbortNode(state, "Failed to write block"); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); @@ -3401,15 +3420,17 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { - bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + DisconnectResult res = DisconnectBlock(block, pindex, coins); + if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } pindexState = pindex->pprev; - if (!fClean) { + if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; - } else + } else { nGoodTransactions += block.vtx.size(); + } } if (ShutdownRequested()) return true; @@ -3585,7 +3606,7 @@ bool InitBlockIndex(const CChainParams& chainparams) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) return error("LoadBlockIndex(): writing genesis block to disk failed"); CBlockIndex *pindex = AddToBlockIndex(block); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("LoadBlockIndex(): genesis block not accepted"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); diff --git a/src/validation.h b/src/validation.h index 60491818a8..743a2973f8 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,10 +11,10 @@ #endif #include "amount.h" -#include "chain.h" #include "coins.h" #include "fs.h" #include "protocol.h" // For CMessageHeader::MessageStartChars +#include "policy/feerate.h" #include "script/script_error.h" #include "sync.h" #include "versionbits.h" @@ -39,6 +39,7 @@ class CChainParams; class CInv; class CConnman; class CScriptCheck; +class CBlockPolicyEstimator; class CTxMemPool; class CValidationInterface; class CValidationState; @@ -131,7 +132,8 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; - +/** Default for -persistmempool */ +static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -mempoolreplacement */ static const bool DEFAULT_ENABLE_REPLACEMENT = true; /** Default for using fee filter */ @@ -145,6 +147,9 @@ static const int MAX_UNCONNECTING_HEADERS = 10; static const bool DEFAULT_PEERBLOOMFILTERS = true; +/** Default for -stopatheight */ +static const int DEFAULT_STOPATHEIGHT = 0; + struct BlockHasher { size_t operator()(const uint256& hash) const { return hash.GetCheapHash(); } @@ -152,6 +157,7 @@ struct BlockHasher extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; +extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef boost::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap mapBlockIndex; @@ -433,18 +439,6 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); -/** Apply the effects of this block (with given index) on the UTXO set represented by coins. - * Validity checks that depend on the UTXO set are also done; ConnectBlock() - * can fail if those validity checks fail (among other reasons). */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, - const CChainParams& chainparams, bool fJustCheck = false); - -/** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& coins, bool* pfClean = NULL); - /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index d4121a28bc..46d7c9b329 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -14,39 +14,39 @@ CMainSignals& GetMainSignals() void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); - g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); - g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); + g_signals.TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.ScriptForMining.connect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); - g_signals.BlockFound.connect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.NewPoWValidBlock.connect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.BlockFound.disconnect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.ScriptForMining.disconnect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); g_signals.BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.SyncTransaction.disconnect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2, _3)); + g_signals.TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); + g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); + g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterAllValidationInterfaces() { - g_signals.BlockFound.disconnect_all_slots(); g_signals.ScriptForMining.disconnect_all_slots(); g_signals.BlockChecked.disconnect_all_slots(); g_signals.Broadcast.disconnect_all_slots(); g_signals.Inventory.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); - g_signals.SyncTransaction.disconnect_all_slots(); + g_signals.TransactionAddedToMempool.disconnect_all_slots(); + g_signals.BlockConnected.disconnect_all_slots(); + g_signals.BlockDisconnected.disconnect_all_slots(); g_signals.UpdatedBlockTip.disconnect_all_slots(); g_signals.NewPoWValidBlock.disconnect_all_slots(); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 7f13a29d23..460aecf243 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -7,16 +7,16 @@ #define BITCOIN_VALIDATIONINTERFACE_H #include <boost/signals2/signal.hpp> -#include <boost/shared_ptr.hpp> #include <memory> +#include "primitives/transaction.h" // CTransaction(Ref) + class CBlock; class CBlockIndex; struct CBlockLocator; class CBlockIndex; class CConnman; class CReserveScript; -class CTransaction; class CValidationInterface; class CValidationState; class uint256; @@ -33,14 +33,14 @@ void UnregisterAllValidationInterfaces(); class CValidationInterface { protected: virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - virtual void SyncTransaction(const CTransaction &tx, const CBlockIndex *pindex, int posInBlock) {} + virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} + virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} virtual void SetBestChain(const CBlockLocator &locator) {} - virtual void UpdatedTransaction(const uint256 &hash) {} virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} virtual void BlockChecked(const CBlock&, const CValidationState&) {} - virtual void GetScriptForMining(boost::shared_ptr<CReserveScript>&) {}; - virtual void ResetRequestCount(const uint256 &hash) {}; + virtual void GetScriptForMining(std::shared_ptr<CReserveScript>&) {}; virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); @@ -50,19 +50,15 @@ protected: struct CMainSignals { /** Notifies listeners of updated block chain tip */ boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; - /** A posInBlock value for SyncTransaction calls for transactions not - * included in connected blocks such as transactions removed from mempool, - * accepted to mempool or appearing in disconnected blocks.*/ - static const int SYNC_TRANSACTION_NOT_IN_BLOCK = -1; - /** Notifies listeners of updated transaction data (transaction, and - * optionally the block it is found in). Called with block data when - * transaction is included in a connected block, and without block data when - * transaction was accepted to mempool, removed from mempool (only when - * removal was due to conflict from connected block), or appeared in a - * disconnected block.*/ - boost::signals2::signal<void (const CTransaction &, const CBlockIndex *pindex, int posInBlock)> SyncTransaction; - /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ - boost::signals2::signal<void (const uint256 &)> UpdatedTransaction; + /** Notifies listeners of a transaction having been added to mempool. */ + boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; + /** + * Notifies listeners of a block being connected. + * Provides a vector of transactions evicted from the mempool as a result. + */ + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &)> BlockConnected; + /** Notifies listeners of a block being disconnected */ + boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; /** Notifies listeners of a new active block chain. */ boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; /** Notifies listeners about an inventory item being seen on the network. */ @@ -77,9 +73,7 @@ struct CMainSignals { */ boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; /** Notifies listeners that a key for mining is required (coinbase) */ - boost::signals2::signal<void (boost::shared_ptr<CReserveScript>&)> ScriptForMining; - /** Notifies listeners that a block has been successfully mined */ - boost::signals2::signal<void (const uint256 &)> BlockFound; + boost::signals2::signal<void (std::shared_ptr<CReserveScript>&)> ScriptForMining; /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 4e93e929be..2aa26fb00a 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include "policy/feerate.h" #include "primitives/transaction.h" #include "wallet/wallet.h" diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index f47fc92b57..25f6bdd9d9 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -359,13 +359,16 @@ void CDBEnv::CheckpointLSN(const std::string& strFile) } -CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) +CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) { int ret; fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; - if (strFilename.empty()) + env = dbw.env; + if (dbw.IsDummy()) { return; + } + const std::string &strFilename = dbw.strFile; bool fCreate = strchr(pszMode, 'c') != NULL; unsigned int nFlags = DB_THREAD; @@ -373,17 +376,17 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose nFlags |= DB_CREATE; { - LOCK(bitdb.cs_db); - if (!bitdb.Open(GetDataDir())) + LOCK(env->cs_db); + if (!env->Open(GetDataDir())) throw std::runtime_error("CDB: Failed to open database environment."); strFile = strFilename; - ++bitdb.mapFileUseCount[strFile]; - pdb = bitdb.mapDb[strFile]; + ++env->mapFileUseCount[strFile]; + pdb = env->mapDb[strFile]; if (pdb == NULL) { - pdb = new Db(bitdb.dbenv, 0); + pdb = new Db(env->dbenv, 0); - bool fMockDb = bitdb.IsMock(); + bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile* mpf = pdb->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); @@ -401,7 +404,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose if (ret != 0) { delete pdb; pdb = NULL; - --bitdb.mapFileUseCount[strFile]; + --env->mapFileUseCount[strFile]; strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } @@ -413,7 +416,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose fReadOnly = fTmp; } - bitdb.mapDb[strFile] = pdb; + env->mapDb[strFile] = pdb; } } } @@ -428,7 +431,7 @@ void CDB::Flush() if (fReadOnly) nMinutes = 1; - bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } void CDB::Close() @@ -444,8 +447,8 @@ void CDB::Close() Flush(); { - LOCK(bitdb.cs_db); - --bitdb.mapFileUseCount[strFile]; + LOCK(env->cs_db); + --env->mapFileUseCount[strFile]; } } @@ -463,32 +466,28 @@ void CDBEnv::CloseDb(const std::string& strFile) } } -bool CDBEnv::RemoveDb(const std::string& strFile) -{ - this->CloseDb(strFile); - - LOCK(cs_db); - int rc = dbenv->dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT); - return (rc == 0); -} - -bool CDB::Rewrite(const std::string& strFile, const char* pszSkip) +bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) { + if (dbw.IsDummy()) { + return true; + } + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; while (true) { { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} - CDB db(strFile.c_str(), "r"); - Db* pdbCopy = new Db(bitdb.dbenv, 0); + CDB db(dbw, "r"); + Db* pdbCopy = new Db(env->dbenv, 0); int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename @@ -531,17 +530,17 @@ bool CDB::Rewrite(const std::string& strFile, const char* pszSkip) } if (fSuccess) { db.Close(); - bitdb.CloseDb(strFile); + env->CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; } } if (fSuccess) { - Db dbA(bitdb.dbenv, 0); + Db dbA(env->dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; - Db dbB(bitdb.dbenv, 0); + Db dbB(env->dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } @@ -596,16 +595,21 @@ void CDBEnv::Flush(bool fShutdown) } } -bool CDB::PeriodicFlush(std::string strFile) +bool CDB::PeriodicFlush(CWalletDBWrapper& dbw) { + if (dbw.IsDummy()) { + return true; + } bool ret = false; + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; TRY_LOCK(bitdb.cs_db,lockDb); if (lockDb) { // Don't do this if any databases are in use int nRefCount = 0; - std::map<std::string, int>::iterator mit = bitdb.mapFileUseCount.begin(); - while (mit != bitdb.mapFileUseCount.end()) + std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); + while (mit != env->mapFileUseCount.end()) { nRefCount += (*mit).second; mit++; @@ -614,17 +618,17 @@ bool CDB::PeriodicFlush(std::string strFile) if (nRefCount == 0) { boost::this_thread::interruption_point(); - std::map<std::string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile); - if (mi != bitdb.mapFileUseCount.end()) + std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); + if (mi != env->mapFileUseCount.end()) { LogPrint(BCLog::DB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(mi++); + env->mapFileUseCount.erase(mi++); LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } @@ -633,3 +637,52 @@ bool CDB::PeriodicFlush(std::string strFile) return ret; } + +bool CWalletDBWrapper::Rewrite(const char* pszSkip) +{ + return CDB::Rewrite(*this, pszSkip); +} + +bool CWalletDBWrapper::Backup(const std::string& strDest) +{ + if (IsDummy()) { + return false; + } + while (true) + { + { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) + { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); + + // Copy wallet file + fs::path pathSrc = GetDataDir() / strFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) + pathDest /= strFile; + + try { + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); + LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what()); + return false; + } + } + } + MilliSleep(100); + } + return false; +} + +void CWalletDBWrapper::Flush(bool shutdown) +{ + if (!IsDummy()) { + env->Flush(shutdown); + } +} diff --git a/src/wallet/db.h b/src/wallet/db.h index 9f912f9a1a..3c6870d169 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -72,7 +72,6 @@ public: void CheckpointLSN(const std::string& strFile); void CloseDb(const std::string& strFile); - bool RemoveDb(const std::string& strFile); DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { @@ -86,6 +85,52 @@ public: extern CDBEnv bitdb; +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class CWalletDBWrapper +{ + friend class CDB; +public: + /** Create dummy DB handle */ + CWalletDBWrapper(): env(nullptr) + { + } + + /** Create DB handle to real database */ + CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in): + env(env_in), strFile(strFile_in) + { + } + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr); + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest); + + /** Get a name for this database, for debugging etc. + */ + std::string GetName() const { return strFile; } + + /** Make sure all changes are flushed to disk. + */ + void Flush(bool shutdown); + +private: + /** BerkeleyDB specific */ + CDBEnv *env; + std::string strFile; + + /** Return whether this database handle is a dummy for testing. + * Only to be used at a low level, application should ideally not care + * about this. + */ + bool IsDummy() { return env == nullptr; } +}; + /** RAII class that provides access to a Berkeley database */ class CDB @@ -96,18 +141,19 @@ protected: DbTxn* activeTxn; bool fReadOnly; bool fFlushOnClose; + CDBEnv *env; - explicit CDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnCloseIn=true); +public: + explicit CDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool fFlushOnCloseIn=true); ~CDB() { Close(); } -public: void Flush(); void Close(); static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); /* flush the wallet passively (TRY_LOCK) ideal to be called periodically */ - static bool PeriodicFlush(std::string strFile); + static bool PeriodicFlush(CWalletDBWrapper& dbw); /* verifies the database environment */ static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); /* verifies the database file */ @@ -117,7 +163,7 @@ private: CDB(const CDB&); void operator=(const CDB&); -protected: +public: template <typename K, typename T> bool Read(const K& key, T& value) { @@ -134,29 +180,30 @@ protected: Dbt datValue; datValue.set_flags(DB_DBT_MALLOC); int ret = pdb->get(activeTxn, &datKey, &datValue, 0); - memset(datKey.get_data(), 0, datKey.get_size()); - if (datValue.get_data() == NULL) - return false; - - // Unserialize value - try { - CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); - ssValue >> value; - } catch (const std::exception&) { - return false; + memory_cleanse(datKey.get_data(), datKey.get_size()); + bool success = false; + if (datValue.get_data() != NULL) { + // Unserialize value + try { + CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); + ssValue >> value; + success = true; + } catch (const std::exception&) { + // In this case success remains 'false' + } + + // Clear and free memory + memory_cleanse(datValue.get_data(), datValue.get_size()); + free(datValue.get_data()); } - - // Clear and free memory - memset(datValue.get_data(), 0, datValue.get_size()); - free(datValue.get_data()); - return (ret == 0); + return ret == 0 && success; } template <typename K, typename T> bool Write(const K& key, const T& value, bool fOverwrite = true) { if (!pdb) - return false; + return true; if (fReadOnly) assert(!"Write called on database in read-only mode"); @@ -176,8 +223,8 @@ protected: int ret = pdb->put(activeTxn, &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); // Clear memory in case it was a private key - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); return (ret == 0); } @@ -199,7 +246,7 @@ protected: int ret = pdb->del(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0 || ret == DB_NOTFOUND); } @@ -219,7 +266,7 @@ protected: int ret = pdb->exists(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0); } @@ -262,8 +309,8 @@ protected: ssValue.write((char*)datValue.get_data(), datValue.get_size()); // Clear and free memory - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); free(datKey.get_data()); free(datValue.get_data()); return 0; @@ -310,7 +357,7 @@ public: return Write(std::string("version"), nVersion); } - bool static Rewrite(const std::string& strFile, const char* pszSkip = NULL); + bool static Rewrite(CWalletDBWrapper& dbw, const char* pszSkip = NULL); }; #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp new file mode 100644 index 0000000000..c10a9eccd9 --- /dev/null +++ b/src/wallet/feebumper.cpp @@ -0,0 +1,296 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "consensus/validation.h" +#include "wallet/feebumper.h" +#include "wallet/wallet.h" +#include "policy/fees.h" +#include "policy/policy.h" +#include "policy/rbf.h" +#include "validation.h" //for mempool access +#include "txmempool.h" +#include "utilmoneystr.h" +#include "util.h" +#include "net.h" + +// Calculate the size of the transaction assuming all signatures are max size +// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. +// TODO: re-use this in CWallet::CreateTransaction (right now +// CreateTransaction uses the constructed dummy-signed tx to do a priority +// calculation, but we should be able to refactor after priority is removed). +// NOTE: this requires that all inputs must be in mapWallet (eg the tx should +// be IsAllFromMe). +int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWallet) +{ + CMutableTransaction txNew(tx); + std::vector<CInputCoin> vCoins; + // Look up the inputs. We should have already checked that this transaction + // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our + // wallet, with a valid index into the vout array. + for (auto& input : tx.vin) { + const auto mi = pWallet->mapWallet.find(input.prevout.hash); + assert(mi != pWallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); + vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n)); + } + if (!pWallet->DummySignTx(txNew, vCoins)) { + // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) + // implies that we can sign for every input. + return -1; + } + return GetVirtualTransactionSize(txNew); +} + +bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx) { + if (pWallet->HasWalletSpend(wtx.GetHash())) { + vErrors.push_back("Transaction has descendants in the wallet"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + + { + LOCK(mempool.cs); + auto it_mp = mempool.mapTx.find(wtx.GetHash()); + if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + vErrors.push_back("Transaction has descendants in the mempool"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + } + + if (wtx.GetDepthInMainChain() != 0) { + vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + currentResult = BumpFeeResult::WALLET_ERROR; + return false; + } + return true; +} + +CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable) + : + txid(std::move(txidIn)), + nOldFee(0), + nNewFee(0) +{ + vErrors.clear(); + bumpedTxid.SetNull(); + AssertLockHeld(pWallet->cs_wallet); + if (!pWallet->mapWallet.count(txid)) { + vErrors.push_back("Invalid or non-wallet transaction id"); + currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; + return; + } + auto it = pWallet->mapWallet.find(txid); + const CWalletTx& wtx = it->second; + + if (!preconditionChecks(pWallet, wtx)) { + return; + } + + if (!SignalsOptInRBF(wtx)) { + vErrors.push_back("Transaction is not BIP 125 replaceable"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + if (wtx.mapValue.count("replaced_by_txid")) { + vErrors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // check that original tx consists entirely of our inputs + // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) + if (!pWallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { + vErrors.push_back("Transaction contains inputs that don't belong to this wallet"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // figure out which output was change + // if there was no change output or multiple change outputs, fail + int nOutput = -1; + for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { + if (pWallet->IsChange(wtx.tx->vout[i])) { + if (nOutput != -1) { + vErrors.push_back("Transaction has multiple change outputs"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + nOutput = i; + } + } + if (nOutput == -1) { + vErrors.push_back("Transaction does not have a change output"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // Calculate the expected size of the new transaction. + int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); + const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, pWallet); + if (maxNewTxSize < 0) { + vErrors.push_back("Transaction contains inputs that cannot be signed"); + currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; + return; + } + + // calculate the old fee and fee-rate + nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + CFeeRate nOldFeeRate(nOldFee, txSize); + CFeeRate nNewFeeRate; + // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to + // future proof against changes to network wide policy for incremental relay + // fee that our node may not be aware of. + CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); + if (::incrementalRelayFee > walletIncrementalRelayFee) { + walletIncrementalRelayFee = ::incrementalRelayFee; + } + + if (totalFee > 0) { + CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); + if (totalFee < minTotalFee) { + vErrors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", + FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + CAmount requiredFee = CWallet::GetRequiredFee(maxNewTxSize); + if (totalFee < requiredFee) { + vErrors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", + FormatMoney(requiredFee))); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return; + } + nNewFee = totalFee; + nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); + } else { + // if user specified a confirm target then don't consider any global payTxFee + if (specifiedConfirmTarget) { + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, true); + } + // otherwise use the regular wallet logic to select payTxFee or default confirm target + else { + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator); + } + + nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); + + // New fee rate must be at least old rate + minimum incremental relay rate + // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized + // in that unit (fee per kb). + // However, nOldFeeRate is a calculated value from the tx fee/size, so + // add 1 satoshi to the result, because it may have been rounded down. + if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { + nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); + nNewFee = nNewFeeRate.GetFee(maxNewTxSize); + } + } + + // Check that in all cases the new fee doesn't violate maxTxFee + if (nNewFee > maxTxFee) { + vErrors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", + FormatMoney(nNewFee), FormatMoney(maxTxFee))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // check that fee rate is higher than mempool's minimum fee + // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) + // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, + // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a + // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. + CFeeRate minMempoolFeeRate = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { + vErrors.push_back(strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // Now modify the output to increase the fee. + // If the output is not large enough to pay the fee, fail. + CAmount nDelta = nNewFee - nOldFee; + assert(nDelta > 0); + mtx = *wtx.tx; + CTxOut* poutput = &(mtx.vout[nOutput]); + if (poutput->nValue < nDelta) { + vErrors.push_back("Change output is too small to bump the fee"); + currentResult = BumpFeeResult::WALLET_ERROR; + return; + } + + // If the output would become dust, discard it (converting the dust to fee) + poutput->nValue -= nDelta; + if (poutput->nValue <= GetDustThreshold(*poutput, ::dustRelayFee)) { + LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); + nNewFee += poutput->nValue; + mtx.vout.erase(mtx.vout.begin() + nOutput); + } + + // Mark new tx not replaceable, if requested. + if (!newTxReplaceable) { + for (auto& input : mtx.vin) { + if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; + } + } + + currentResult = BumpFeeResult::OK; +} + +bool CFeeBumper::signTransaction(CWallet *pWallet) +{ + return pWallet->SignTransaction(mtx); +} + +bool CFeeBumper::commit(CWallet *pWallet) +{ + AssertLockHeld(pWallet->cs_wallet); + if (!vErrors.empty() || currentResult != BumpFeeResult::OK) { + return false; + } + if (txid.IsNull() || !pWallet->mapWallet.count(txid)) { + vErrors.push_back("Invalid or non-wallet transaction id"); + currentResult = BumpFeeResult::MISC_ERROR; + return false; + } + CWalletTx& oldWtx = pWallet->mapWallet[txid]; + + // make sure the transaction still has no descendants and hasen't been mined in the meantime + if (!preconditionChecks(pWallet, oldWtx)) { + return false; + } + + CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); + // commit/broadcast the tx + CReserveKey reservekey(pWallet); + wtxBumped.mapValue = oldWtx.mapValue; + wtxBumped.mapValue["replaces_txid"] = oldWtx.GetHash().ToString(); + wtxBumped.vOrderForm = oldWtx.vOrderForm; + wtxBumped.strFromAccount = oldWtx.strFromAccount; + wtxBumped.fTimeReceivedIsTxTime = true; + wtxBumped.fFromMe = true; + CValidationState state; + if (!pWallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { + // NOTE: CommitTransaction never returns false, so this should never happen. + vErrors.push_back(strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); + return false; + } + + bumpedTxid = wtxBumped.GetHash(); + if (state.IsInvalid()) { + // This can happen if the mempool rejected the transaction. Report + // what happened in the "errors" response. + vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + } + + // mark the original tx as bumped + if (!pWallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { + // TODO: see if JSON-RPC has a standard way of returning a response + // along with an exception. It would be good to return information about + // wtxBumped to the caller even if marking the original transaction + // replaced does not succeed for some reason. + vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + } + return true; +} + diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h new file mode 100644 index 0000000000..f40d05da28 --- /dev/null +++ b/src/wallet/feebumper.h @@ -0,0 +1,59 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_FEEBUMPER_H +#define BITCOIN_WALLET_FEEBUMPER_H + +#include <primitives/transaction.h> + +class CWallet; +class CWalletTx; +class uint256; + +enum class BumpFeeResult +{ + OK, + INVALID_ADDRESS_OR_KEY, + INVALID_REQUEST, + INVALID_PARAMETER, + WALLET_ERROR, + MISC_ERROR, +}; + +class CFeeBumper +{ +public: + CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable); + BumpFeeResult getResult() const { return currentResult; } + const std::vector<std::string>& getErrors() const { return vErrors; } + CAmount getOldFee() const { return nOldFee; } + CAmount getNewFee() const { return nNewFee; } + uint256 getBumpedTxId() const { return bumpedTxid; } + + /* signs the new transaction, + * returns false if the tx couldn't be found or if it was + * impossible to create the signature(s) + */ + bool signTransaction(CWallet *pWallet); + + /* commits the fee bump, + * returns true, in case of CWallet::CommitTransaction was successful + * but, eventually sets vErrors if the tx could not be added to the mempool (will try later) + * or if the old transaction could not be marked as replaced + */ + bool commit(CWallet *pWalletNonConst); + +private: + bool preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx); + + const uint256 txid; + uint256 bumpedTxid; + CMutableTransaction mtx; + std::vector<std::string> vErrors; + BumpFeeResult currentResult; + CAmount nOldFee; + CAmount nNewFee; +}; + +#endif // BITCOIN_WALLET_FEEBUMPER_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 7ff9e7ae58..82708dab26 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -95,6 +95,8 @@ UniValue importprivkey(const JSONRPCRequest& request) + HelpExampleCli("importprivkey", "\"mykey\"") + "\nImport using a label and without rescan\n" + HelpExampleCli("importprivkey", "\"mykey\" \"testing\" false") + + "\nImport using default blank label and without rescan\n" + + HelpExampleCli("importprivkey", "\"mykey\" \"\" false") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") ); @@ -154,6 +156,31 @@ UniValue importprivkey(const JSONRPCRequest& request) return NullUniValue; } +UniValue abortrescan(const JSONRPCRequest& request) +{ + CWallet* const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 0) + throw std::runtime_error( + "abortrescan\n" + "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nExamples:\n" + "\nImport a private key\n" + + HelpExampleCli("importprivkey", "\"mykey\"") + + "\nAbort the running wallet rescan\n" + + HelpExampleCli("abortrescan", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("abortrescan", "") + ); + + if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; + pwallet->AbortRescan(); + return true; +} + void ImportAddress(CWallet*, const CBitcoinAddress& address, const std::string& strLabel); void ImportScript(CWallet* const pwallet, const CScript& script, const std::string& strLabel, bool isRedeemScript) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index ccb744c6b2..ae4f4f37cb 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -9,8 +9,11 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" +#include "wallet/coincontrol.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "rpc/server.h" @@ -18,8 +21,9 @@ #include "timedata.h" #include "util.h" #include "utilmoneystr.h" -#include "wallet.h" -#include "walletdb.h" +#include "wallet/feebumper.h" +#include "wallet/wallet.h" +#include "wallet/walletdb.h" #include <stdint.h> @@ -726,6 +730,9 @@ UniValue getbalance(const JSONRPCRequest& request) if (request.params.size() == 0) return ValueFromAmount(pwallet->GetBalance()); + const std::string& account_param = request.params[0].get_str(); + const std::string* account = account_param != "*" ? &account_param : nullptr; + int nMinDepth = 1; if (request.params.size() > 1) nMinDepth = request.params[1].get_int(); @@ -734,41 +741,7 @@ UniValue getbalance(const JSONRPCRequest& request) if(request.params[2].get_bool()) filter = filter | ISMINE_WATCH_ONLY; - if (request.params[0].get_str() == "*") { - // Calculate total balance in a very different way from GetBalance(). - // The biggest difference is that GetBalance() sums up all unspent - // TxOuts paying to the wallet, while this sums up both spent and - // unspent TxOuts paying to the wallet, and then subtracts the values of - // TxIns spending from the wallet. This also has fewer restrictions on - // which unconfirmed transactions are considered trusted. - CAmount nBalance = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount allFee; - std::string strSentAccount; - std::list<COutputEntry> listReceived; - std::list<COutputEntry> listSent; - wtx.GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - if (wtx.GetDepthInMainChain() >= nMinDepth) - { - BOOST_FOREACH(const COutputEntry& r, listReceived) - nBalance += r.amount; - } - BOOST_FOREACH(const COutputEntry& s, listSent) - nBalance -= s.amount; - nBalance -= allFee; - } - return ValueFromAmount(nBalance); - } - - std::string strAccount = AccountFromValue(request.params[0]); - - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, filter); - - return ValueFromAmount(nBalance); + return ValueFromAmount(pwallet->GetLegacyBalance(filter, nMinDepth, account)); } UniValue getunconfirmedbalance(const JSONRPCRequest &request) @@ -898,7 +871,7 @@ UniValue sendfrom(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (nAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -1007,7 +980,7 @@ UniValue sendmany(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (totalAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -2075,7 +2048,7 @@ UniValue walletpassphrase(const JSONRPCRequest& request) int64_t nSleepTime = request.params[1].get_int64(); pwallet->nRelockTime = GetTime() + nSleepTime; - RPCRunLater(strprintf("lockwallet(%s)", pwallet->strWalletFile), boost::bind(LockWallet, pwallet), nSleepTime); + RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), boost::bind(LockWallet, pwallet), nSleepTime); return NullUniValue; } @@ -2497,22 +2470,29 @@ UniValue listunspent(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() > 4) + if (request.fHelp || request.params.size() > 5) throw std::runtime_error( - "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] )\n" + "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] [query_options])\n" "\nReturns array of unspent transaction outputs\n" "with between minconf and maxconf (inclusive) confirmations.\n" "Optionally filter to only include txouts paid to specified addresses.\n" "\nArguments:\n" "1. minconf (numeric, optional, default=1) The minimum confirmations to filter\n" "2. maxconf (numeric, optional, default=9999999) The maximum confirmations to filter\n" - "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" + "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" " [\n" - " \"address\" (string) bitcoin address\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ]\n" "4. include_unsafe (bool, optional, default=true) Include outputs that are not safe to spend\n" " See description of \"safe\" attribute below.\n" + "5. query_options (json, optional) JSON with query options\n" + " {\n" + " \"minimumAmount\" (numeric or string, default=0) Minimum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumAmount\" (numeric or string, default=unlimited) Maximum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumCount\" (numeric or string, default=unlimited) Maximum number of UTXOs\n" + " \"minimumSumAmount\" (numeric or string, default=unlimited) Minimum sum value of all UTXOs in " + CURRENCY_UNIT + "\n" + " }\n" "\nResult\n" "[ (array of json object)\n" " {\n" @@ -2537,6 +2517,8 @@ UniValue listunspent(const JSONRPCRequest& request) + HelpExampleCli("listunspent", "") + HelpExampleCli("listunspent", "6 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + HelpExampleRpc("listunspent", "6, 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + HelpExampleCli("listunspent", "6 9999999 '[]' true '{ \"minimumAmount\": 0.005 }'") + + HelpExampleRpc("listunspent", "6, 9999999, [] , true, { \"minimumAmount\": 0.005 } ") ); int nMinDepth = 1; @@ -2572,15 +2554,34 @@ UniValue listunspent(const JSONRPCRequest& request) include_unsafe = request.params[3].get_bool(); } + CAmount nMinimumAmount = 0; + CAmount nMaximumAmount = MAX_MONEY; + CAmount nMinimumSumAmount = MAX_MONEY; + uint64_t nMaximumCount = 0; + + if (request.params.size() > 4) { + const UniValue& options = request.params[4].get_obj(); + + if (options.exists("minimumAmount")) + nMinimumAmount = AmountFromValue(options["minimumAmount"]); + + if (options.exists("maximumAmount")) + nMaximumAmount = AmountFromValue(options["maximumAmount"]); + + if (options.exists("minimumSumAmount")) + nMinimumSumAmount = AmountFromValue(options["minimumSumAmount"]); + + if (options.exists("maximumCount")) + nMaximumCount = options["maximumCount"].get_int64(); + } + UniValue results(UniValue::VARR); std::vector<COutput> vecOutputs; assert(pwallet != NULL); LOCK2(cs_main, pwallet->cs_wallet); - pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, true); - BOOST_FOREACH(const COutput& out, vecOutputs) { - if (out.nDepth < nMinDepth || out.nDepth > nMaxDepth) - continue; + pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); + BOOST_FOREACH(const COutput& out, vecOutputs) { CTxDestination address; const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); @@ -2778,33 +2779,6 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) return result; } -// Calculate the size of the transaction assuming all signatures are max size -// Use DummySignatureCreator, which inserts 72 byte signatures everywhere. -// TODO: re-use this in CWallet::CreateTransaction (right now -// CreateTransaction uses the constructed dummy-signed tx to do a priority -// calculation, but we should be able to refactor after priority is removed). -// NOTE: this requires that all inputs must be in mapWallet (eg the tx should -// be IsAllFromMe). -int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, CWallet &wallet) -{ - CMutableTransaction txNew(tx); - std::vector<std::pair<CWalletTx*, unsigned int>> vCoins; - // Look up the inputs. We should have already checked that this transaction - // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our - // wallet, with a valid index into the vout array. - for (auto& input : tx.vin) { - const auto mi = wallet.mapWallet.find(input.prevout.hash); - assert(mi != wallet.mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - vCoins.emplace_back(std::make_pair(&(mi->second), input.prevout.n)); - } - if (!wallet.DummySignTx(txNew, vCoins)) { - // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) - // implies that we can sign for every input. - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction contains inputs that cannot be signed"); - } - return GetVirtualTransactionSize(txNew); -} - UniValue bumpfee(const JSONRPCRequest& request) { CWallet * const pwallet = GetWalletForJSONRPCRequest(request); @@ -2823,7 +2797,7 @@ UniValue bumpfee(const JSONRPCRequest& request) "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" "By default, the new fee will be calculated automatically using estimatefee.\n" "The user can specify a confirmation target for estimatefee.\n" - "Alternatively, the user can specify totalFee, or use RPC setpaytxfee to set a higher fee rate.\n" + "Alternatively, the user can specify totalFee, or use RPC settxfee to set a higher fee rate.\n" "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" "returned by getnetworkinfo) to enter the node's mempool.\n" "\nArguments:\n" @@ -2859,63 +2833,6 @@ UniValue bumpfee(const JSONRPCRequest& request) uint256 hash; hash.SetHex(request.params[0].get_str()); - // retrieve the original tx from the wallet - LOCK2(cs_main, pwallet->cs_wallet); - EnsureWalletIsUnlocked(pwallet); - if (!pwallet->mapWallet.count(hash)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); - } - CWalletTx& wtx = pwallet->mapWallet[hash]; - - if (pwallet->HasWalletSpend(hash)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Transaction has descendants in the wallet"); - } - - { - LOCK(mempool.cs); - auto it = mempool.mapTx.find(hash); - if (it != mempool.mapTx.end() && it->GetCountWithDescendants() > 1) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Transaction has descendants in the mempool"); - } - } - - if (wtx.GetDepthInMainChain() != 0) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction has been mined, or is conflicted with a mined transaction"); - } - - if (!SignalsOptInRBF(wtx)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction is not BIP 125 replaceable"); - } - - if (wtx.mapValue.count("replaced_by_txid")) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Cannot bump transaction %s which was already bumped by transaction %s", hash.ToString(), wtx.mapValue.at("replaced_by_txid"))); - } - - // check that original tx consists entirely of our inputs - // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!pwallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction contains inputs that don't belong to this wallet"); - } - - // figure out which output was change - // if there was no change output or multiple change outputs, fail - int nOutput = -1; - for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { - if (pwallet->IsChange(wtx.tx->vout[i])) { - if (nOutput != -1) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction has multiple change outputs"); - } - nOutput = i; - } - } - if (nOutput == -1) { - throw JSONRPCError(RPC_WALLET_ERROR, "Transaction does not have a change output"); - } - - // Calculate the expected size of the new transaction. - int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); - const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, *pwallet); - // optional parameters bool specifiedConfirmTarget = false; int newConfirmTarget = nTxConfirmTarget; @@ -2941,11 +2858,8 @@ UniValue bumpfee(const JSONRPCRequest& request) } } else if (options.exists("totalFee")) { totalFee = options["totalFee"].get_int64(); - CAmount requiredFee = CWallet::GetRequiredFee(maxNewTxSize); - if (totalFee < requiredFee ) { - throw JSONRPCError(RPC_INVALID_PARAMETER, - strprintf("Insufficient totalFee (cannot be less than required fee %s)", - FormatMoney(requiredFee))); + if (totalFee <= 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid totalFee %s (must be greater than 0)", FormatMoney(totalFee))); } } @@ -2954,148 +2868,53 @@ UniValue bumpfee(const JSONRPCRequest& request) } } - // calculate the old fee and fee-rate - CAmount nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); - CFeeRate nOldFeeRate(nOldFee, txSize); - CAmount nNewFee; - CFeeRate nNewFeeRate; - // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to - // future proof against changes to network wide policy for incremental relay - // fee that our node may not be aware of. - CFeeRate walletIncrementalRelayFee = CFeeRate(WALLET_INCREMENTAL_RELAY_FEE); - if (::incrementalRelayFee > walletIncrementalRelayFee) { - walletIncrementalRelayFee = ::incrementalRelayFee; - } - - if (totalFee > 0) { - CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); - if (totalFee < minTotalFee) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", - FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); - } - nNewFee = totalFee; - nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); - } else { - // if user specified a confirm target then don't consider any global payTxFee - if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, CAmount(0)); - } - // otherwise use the regular wallet logic to select payTxFee or default confirm target - else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool); - } - - nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); - - // New fee rate must be at least old rate + minimum incremental relay rate - // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized - // in that unit (fee per kb). - // However, nOldFeeRate is a calculated value from the tx fee/size, so - // add 1 satoshi to the result, because it may have been rounded down. - if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { - nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); - nNewFee = nNewFeeRate.GetFee(maxNewTxSize); - } - } - - // Check that in all cases the new fee doesn't violate maxTxFee - if (nNewFee > maxTxFee) { - throw JSONRPCError(RPC_WALLET_ERROR, - strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(nNewFee), FormatMoney(maxTxFee))); - } - - // check that fee rate is higher than mempool's minimum fee - // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) - // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, - // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a - // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. - CFeeRate minMempoolFeeRate = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); - if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); - } - - // Now modify the output to increase the fee. - // If the output is not large enough to pay the fee, fail. - CAmount nDelta = nNewFee - nOldFee; - assert(nDelta > 0); - CMutableTransaction tx(*(wtx.tx)); - CTxOut* poutput = &(tx.vout[nOutput]); - if (poutput->nValue < nDelta) { - throw JSONRPCError(RPC_WALLET_ERROR, "Change output is too small to bump the fee"); - } - - // If the output would become dust, discard it (converting the dust to fee) - poutput->nValue -= nDelta; - if (poutput->nValue <= poutput->GetDustThreshold(::dustRelayFee)) { - LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); - nNewFee += poutput->nValue; - tx.vout.erase(tx.vout.begin() + nOutput); - } - - // Mark new tx not replaceable, if requested. - if (!replaceable) { - for (auto& input : tx.vin) { - if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; - } - } + LOCK2(cs_main, pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); - // sign the new tx - CTransaction txNewConst(tx); - int nIn = 0; - for (auto& input : tx.vin) { - std::map<uint256, CWalletTx>::const_iterator mi = pwallet->mapWallet.find(input.prevout.hash); - assert(mi != pwallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; - const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; - SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(pwallet, &txNewConst, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); + CFeeBumper feeBump(pwallet, hash, newConfirmTarget, specifiedConfirmTarget, totalFee, replaceable); + BumpFeeResult res = feeBump.getResult(); + if (res != BumpFeeResult::OK) + { + switch(res) { + case BumpFeeResult::INVALID_ADDRESS_OR_KEY: + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, feeBump.getErrors()[0]); + break; + case BumpFeeResult::INVALID_REQUEST: + throw JSONRPCError(RPC_INVALID_REQUEST, feeBump.getErrors()[0]); + break; + case BumpFeeResult::INVALID_PARAMETER: + throw JSONRPCError(RPC_INVALID_PARAMETER, feeBump.getErrors()[0]); + break; + case BumpFeeResult::WALLET_ERROR: + throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + break; + default: + throw JSONRPCError(RPC_MISC_ERROR, feeBump.getErrors()[0]); + break; } - UpdateTransaction(tx, nIn, sigdata); - nIn++; } - // commit/broadcast the tx - CReserveKey reservekey(pwallet); - CWalletTx wtxBumped(pwallet, MakeTransactionRef(std::move(tx))); - wtxBumped.mapValue = wtx.mapValue; - wtxBumped.mapValue["replaces_txid"] = hash.ToString(); - wtxBumped.vOrderForm = wtx.vOrderForm; - wtxBumped.strFromAccount = wtx.strFromAccount; - wtxBumped.fTimeReceivedIsTxTime = true; - wtxBumped.fFromMe = true; - CValidationState state; - if (!pwallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { - // NOTE: CommitTransaction never returns false, so this should never happen. - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); + // sign bumped transaction + if (!feeBump.signTransaction(pwallet)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } - - UniValue vErrors(UniValue::VARR); - if (state.IsInvalid()) { - // This can happen if the mempool rejected the transaction. Report - // what happened in the "errors" response. - vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); - } - - // mark the original tx as bumped - if (!pwallet->MarkReplaced(wtx.GetHash(), wtxBumped.GetHash())) { - // TODO: see if JSON-RPC has a standard way of returning a response - // along with an exception. It would be good to return information about - // wtxBumped to the caller even if marking the original transaction - // replaced does not succeed for some reason. - vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + // commit the bumped transaction + if(!feeBump.commit(pwallet)) { + throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); } - UniValue result(UniValue::VOBJ); - result.push_back(Pair("txid", wtxBumped.GetHash().GetHex())); - result.push_back(Pair("origfee", ValueFromAmount(nOldFee))); - result.push_back(Pair("fee", ValueFromAmount(nNewFee))); - result.push_back(Pair("errors", vErrors)); + result.push_back(Pair("txid", feeBump.getBumpedTxId().GetHex())); + result.push_back(Pair("origfee", ValueFromAmount(feeBump.getOldFee()))); + result.push_back(Pair("fee", ValueFromAmount(feeBump.getNewFee()))); + UniValue errors(UniValue::VARR); + for (const std::string& err: feeBump.getErrors()) + errors.push_back(err); + result.push_back(errors); return result; } +extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); extern UniValue importaddress(const JSONRPCRequest& request); @@ -3112,6 +2931,7 @@ static const CRPCCommand commands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false, {"hexstring","options"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, true, {} }, { "wallet", "abandontransaction", &abandontransaction, false, {"txid"} }, + { "wallet", "abortrescan", &abortrescan, false, {} }, { "wallet", "addmultisigaddress", &addmultisigaddress, true, {"nrequired","keys","account"} }, { "wallet", "addwitnessaddress", &addwitnessaddress, true, {"address"} }, { "wallet", "backupwallet", &backupwallet, true, {"destination"} }, @@ -3144,7 +2964,7 @@ static const CRPCCommand commands[] = { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, false, {"minconf","include_empty","include_watchonly"} }, { "wallet", "listsinceblock", &listsinceblock, false, {"blockhash","target_confirmations","include_watchonly"} }, { "wallet", "listtransactions", &listtransactions, false, {"account","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe"} }, + { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, { "wallet", "lockunspent", &lockunspent, true, {"unlock","transactions"} }, { "wallet", "move", &movecmd, false, {"fromaccount","toaccount","amount","minconf","comment"} }, { "wallet", "sendfrom", &sendfrom, false, {"fromaccount","toaddress","amount","minconf","comment","comment_to"} }, diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index a76db37617..1989bf8d9b 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -14,7 +14,8 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): bitdb.MakeMock(); bool fFirstRun; - pwalletMain = new CWallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + pwalletMain = new CWallet(std::move(dbw)); pwalletMain->LoadWallet(fFirstRun); RegisterValidationInterface(pwalletMain); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 67e5e90224..8eeba72a06 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -29,7 +29,7 @@ extern UniValue importmulti(const JSONRPCRequest& request); std::vector<std::unique_ptr<CWalletTx>> wtxn; -typedef std::set<std::pair<const CWalletTx*,unsigned int> > CoinSet; +typedef std::set<CInputCoin> CoinSet; BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) @@ -424,6 +424,17 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); ::pwalletMain = backup; } + + // Verify ScanForWalletTransactions does not return null when the scan is + // elided due to the nTimeFirstKey optimization. + { + CWallet wallet; + { + LOCK(wallet.cs_wallet); + wallet.UpdateTimeFirstKey(newTip->GetBlockTime() + 7200 + 1); + } + BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(newTip)); + } } // Check that GetImmatureCredit() returns a newly calculated value instead of diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 55d81daab4..9eba8ad9fe 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -16,7 +16,9 @@ #include "keystore.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" +#include "policy/rbf.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "script/script.h" @@ -64,10 +66,10 @@ const uint256 CMerkleTx::ABANDON_HASH(uint256S("00000000000000000000000000000000 struct CompareValueOnly { - bool operator()(const std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> >& t1, - const std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> >& t2) const + bool operator()(const CInputCoin& t1, + const CInputCoin& t2) const { - return t1.first < t2.first; + return t1.txout.nValue < t2.txout.nValue; } }; @@ -160,7 +162,7 @@ void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool inter secret = childKey.key; metadata.hdMasterKeyID = hdChain.masterKeyID; // update the chain model in the database - if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + if (!CWalletDB(*dbw).WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } @@ -179,10 +181,8 @@ bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) if (HaveWatchOnly(script)) RemoveWatchOnly(script); - if (!fFileBacked) - return true; if (!IsCrypted()) { - return CWalletDB(strWalletFile).WriteKey(pubkey, + return CWalletDB(*dbw).WriteKey(pubkey, secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } @@ -194,8 +194,6 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, { if (!CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret)) return false; - if (!fFileBacked) - return true; { LOCK(cs_wallet); if (pwalletdbEncryption) @@ -203,7 +201,7 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); else - return CWalletDB(strWalletFile).WriteCryptedKey(vchPubKey, + return CWalletDB(*dbw).WriteCryptedKey(vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); } @@ -239,9 +237,7 @@ bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteCScript(Hash160(redeemScript), redeemScript); + return CWalletDB(*dbw).WriteCScript(Hash160(redeemScript), redeemScript); } bool CWallet::LoadCScript(const CScript& redeemScript) @@ -267,9 +263,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) const CKeyMetadata& meta = mapKeyMetadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteWatchOnly(dest, meta); + return CWalletDB(*dbw).WriteWatchOnly(dest, meta); } bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) @@ -285,9 +279,8 @@ bool CWallet::RemoveWatchOnly(const CScript &dest) return false; if (!HaveWatchOnly()) NotifyWatchonlyChanged(false); - if (fFileBacked) - if (!CWalletDB(strWalletFile).EraseWatchOnly(dest)) - return false; + if (!CWalletDB(*dbw).EraseWatchOnly(dest)) + return false; return true; } @@ -352,7 +345,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; if (!crypter.Encrypt(_vMasterKey, pMasterKey.second.vchCryptedKey)) return false; - CWalletDB(strWalletFile).WriteMasterKey(pMasterKey.first, pMasterKey.second); + CWalletDB(*dbw).WriteMasterKey(pMasterKey.first, pMasterKey.second); if (fWasLocked) Lock(); return true; @@ -365,7 +358,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::SetBestChain(const CBlockLocator& loc) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); walletdb.WriteBestBlock(loc); } @@ -384,9 +377,8 @@ bool CWallet::SetMinVersion(enum WalletFeature nVersion, CWalletDB* pwalletdbIn, if (nVersion > nWalletMaxVersion) nWalletMaxVersion = nVersion; - if (fFileBacked) { - CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(strWalletFile); + CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(*dbw); if (nWalletVersion > 40000) pwalletdb->WriteMinVersion(nWalletVersion); if (!pwalletdbIn) @@ -440,7 +432,7 @@ bool CWallet::HasWalletSpend(const uint256& txid) const void CWallet::Flush(bool shutdown) { - bitdb.Flush(shutdown); + dbw->Flush(shutdown); } bool CWallet::Verify() @@ -593,24 +585,19 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) { LOCK(cs_wallet); mapMasterKeys[++nMasterKeyMaxID] = kMasterKey; - if (fFileBacked) - { - assert(!pwalletdbEncryption); - pwalletdbEncryption = new CWalletDB(strWalletFile); - if (!pwalletdbEncryption->TxnBegin()) { - delete pwalletdbEncryption; - pwalletdbEncryption = NULL; - return false; - } - pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); + assert(!pwalletdbEncryption); + pwalletdbEncryption = new CWalletDB(*dbw); + if (!pwalletdbEncryption->TxnBegin()) { + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + return false; } + pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); if (!EncryptKeys(_vMasterKey)) { - if (fFileBacked) { - pwalletdbEncryption->TxnAbort(); - delete pwalletdbEncryption; - } + pwalletdbEncryption->TxnAbort(); + delete pwalletdbEncryption; // We now probably have half of our keys encrypted in memory, and half not... // die and let the user reload the unencrypted wallet. assert(false); @@ -619,30 +606,24 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Encryption was introduced in version 0.4.0 SetMinVersion(FEATURE_WALLETCRYPT, pwalletdbEncryption, true); - if (fFileBacked) - { - if (!pwalletdbEncryption->TxnCommit()) { - delete pwalletdbEncryption; - // We now have keys encrypted in memory, but not on disk... - // die to avoid confusion and let the user reload the unencrypted wallet. - assert(false); - } - + if (!pwalletdbEncryption->TxnCommit()) { delete pwalletdbEncryption; - pwalletdbEncryption = NULL; + // We now have keys encrypted in memory, but not on disk... + // die to avoid confusion and let the user reload the unencrypted wallet. + assert(false); } + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + Lock(); Unlock(strWalletPassphrase); // if we are using HD, replace the HD master key (seed) with a new one if (IsHDEnabled()) { - CKey key; - CPubKey masterPubKey = GenerateNewHDMasterKey(); - // preserve the old chains version to not break backward compatibility - CHDChain oldChain = GetHDChain(); - if (!SetHDMasterKey(masterPubKey, &oldChain)) + if (!SetHDMasterKey(GenerateNewHDMasterKey())) { return false; + } } NewKeyPool(); @@ -650,7 +631,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Need to completely rewrite the wallet file; if we don't, bdb might keep // bits of the unencrypted private key in slack space in the database file. - CDB::Rewrite(strWalletFile); + dbw->Rewrite(); } NotifyStatusChanged(this); @@ -661,7 +642,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // Old wallets didn't have any defined order for transactions // Probably a bad idea to change the output of this @@ -742,14 +723,14 @@ int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) if (pwalletdb) { pwalletdb->WriteOrderPosNext(nOrderPosNext); } else { - CWalletDB(strWalletFile).WriteOrderPosNext(nOrderPosNext); + CWalletDB(*dbw).WriteOrderPosNext(nOrderPosNext); } return nRet; } bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (!walletdb.TxnBegin()) return false; @@ -783,7 +764,7 @@ bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmoun bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); CAccount account; walletdb.ReadAccount(strAccount, account); @@ -844,7 +825,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) wtx.mapValue["replaced_by_txid"] = newHash.ToString(); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); bool success = true; if (!walletdb.WriteTx(wtx)) { @@ -861,7 +842,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile, "r+", fFlushOnClose); + CWalletDB walletdb(*dbw, "r+", fFlushOnClose); uint256 hash = wtxIn.GetHash(); @@ -956,9 +937,9 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) /** * Add a transaction to the wallet, or update it. pIndex and posInBlock should * be set when the transaction was known to be included in a block. When - * posInBlock = SYNC_TRANSACTION_NOT_IN_BLOCK (-1) , then wallet state is not - * updated in AddToWallet, but notifications happen and cached balances are - * marked dirty. + * pIndex == NULL, then wallet state is not updated in AddToWallet, but + * notifications happen and cached balances are marked dirty. + * * If fUpdate is true, existing transactions will be updated. * TODO: One exception to this is that the abandoned state is cleared under the * assumption that any further notification of a transaction that was considered @@ -966,12 +947,13 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. */ -bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate) { + const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (posInBlock != -1) { + if (pIndex != NULL) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { @@ -988,10 +970,10 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex if (fExisted && !fUpdate) return false; if (fExisted || IsMine(tx) || IsFromMe(tx)) { - CWalletTx wtx(this, MakeTransactionRef(tx)); + CWalletTx wtx(this, ptx); // Get merkle branch if transaction was found in a block - if (posInBlock != -1) + if (pIndex != NULL) wtx.SetMerkleBranch(pIndex, posInBlock); return AddToWallet(wtx, false); @@ -1004,7 +986,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); std::set<uint256> todo; std::set<uint256> done; @@ -1076,7 +1058,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) return; // Do not flush the wallet here for performance reasons - CWalletDB walletdb(strWalletFile, "r+", false); + CWalletDB walletdb(*dbw, "r+", false); std::set<uint256> todo; std::set<uint256> done; @@ -1116,11 +1098,10 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock) -{ - LOCK2(cs_main, cs_wallet); +void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pindex, int posInBlock) { + const CTransaction& tx = *ptx; - if (!AddToWalletIfInvolvingMe(tx, pindex, posInBlock, true)) + if (!AddToWalletIfInvolvingMe(ptx, pindex, posInBlock, true)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1133,6 +1114,38 @@ void CWallet::SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, } } +void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { + LOCK2(cs_main, cs_wallet); + SyncTransaction(ptx); +} + +void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { + LOCK2(cs_main, cs_wallet); + // TODO: Temporarily ensure that mempool removals are notified before + // connected transactions. This shouldn't matter, but the abandoned + // state of transactions in our wallet is currently cleared when we + // receive another notification and there is a race condition where + // notification of a connected conflict might cause an outside process + // to abandon a transaction and then have it inadvertently cleared by + // the notification that the conflicted transaction was evicted. + + for (const CTransactionRef& ptx : vtxConflicted) { + SyncTransaction(ptx); + } + for (size_t i = 0; i < pblock->vtx.size(); i++) { + SyncTransaction(pblock->vtx[i], pindex, i); + } +} + +void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { + LOCK2(cs_main, cs_wallet); + + for (const CTransactionRef& ptx : pblock->vtx) { + SyncTransaction(ptx); + } +} + + isminetype CWallet::IsMine(const CTxIn &txin) const { @@ -1308,17 +1321,14 @@ CPubKey CWallet::GenerateNewHDMasterKey() return pubkey; } -bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) +bool CWallet::SetHDMasterKey(const CPubKey& pubkey) { LOCK(cs_wallet); // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object CHDChain newHdChain; - if (possibleOldChain) { - // preserve the old chains version - newHdChain.nVersion = possibleOldChain->nVersion; - } + newHdChain.nVersion = CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; newHdChain.masterKeyID = pubkey.GetID(); SetHDChain(newHdChain, false); @@ -1328,7 +1338,7 @@ bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) bool CWallet::SetHDChain(const CHDChain& chain, bool memonly) { LOCK(cs_wallet); - if (!memonly && !CWalletDB(strWalletFile).WriteHDChain(chain)) + if (!memonly && !CWalletDB(*dbw).WriteHDChain(chain)) throw std::runtime_error(std::string(__func__) + ": writing chain failed"); hdChain = chain; @@ -1441,59 +1451,27 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, } -void CWalletTx::GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const -{ - nReceived = nSent = nFee = 0; - - CAmount allFee; - std::string strSentAccount; - std::list<COutputEntry> listReceived; - std::list<COutputEntry> listSent; - GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - - if (strAccount == strSentAccount) - { - BOOST_FOREACH(const COutputEntry& s, listSent) - nSent += s.amount; - nFee = allFee; - } - { - LOCK(pwallet->cs_wallet); - BOOST_FOREACH(const COutputEntry& r, listReceived) - { - if (pwallet->mapAddressBook.count(r.destination)) - { - std::map<CTxDestination, CAddressBookData>::const_iterator mi = pwallet->mapAddressBook.find(r.destination); - if (mi != pwallet->mapAddressBook.end() && (*mi).second.name == strAccount) - nReceived += r.amount; - } - else if (strAccount.empty()) - { - nReceived += r.amount; - } - } - } -} - /** * Scan the block chain (starting in pindexStart) for transactions * from or to us. If fUpdate is true, found transactions that already * exist in the wallet will be updated. * * Returns pointer to the first block in the last contiguous range that was - * successfully scanned. - * + * successfully scanned or elided (elided if pIndexStart points at a block + * before CWallet::nTimeFirstKey). Returns null if there is no such range, or + * the range doesn't include chainActive.Tip(). */ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) { - CBlockIndex* ret = nullptr; int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); CBlockIndex* pindex = pindexStart; + CBlockIndex* ret = pindexStart; { LOCK2(cs_main, cs_wallet); + fAbortRescan = false; + fScanningWallet = true; // no need to read and scan block, if block was created before // our wallet birthday (as adjusted for block time variability) @@ -1503,15 +1481,19 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); - while (pindex) + while (pindex && !fAbortRescan) { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + if (GetTime() >= nNow + 60) { + nNow = GetTime(); + LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + } CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - AddToWalletIfInvolvingMe(*block.vtx[posInBlock], pindex, posInBlock, fUpdate); + AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate); } if (!ret) { ret = pindex; @@ -1520,12 +1502,13 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f ret = nullptr; } pindex = chainActive.Next(pindex); - if (GetTime() >= nNow + 60) { - nNow = GetTime(); - LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); - } + } + if (pindex && fAbortRescan) { + LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI + + fScanningWallet = false; } return ret; } @@ -1760,10 +1743,7 @@ CAmount CWalletTx::GetChange() const bool CWalletTx::InMempool() const { LOCK(mempool.cs); - if (mempool.exists(GetHash())) { - return true; - } - return false; + return mempool.exists(GetHash()); } bool CWalletTx::IsTrusted() const @@ -1954,12 +1934,58 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const return nTotal; } -void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl *coinControl, bool fIncludeZeroValue) const +// Calculate total balance in a different way from GetBalance. The biggest +// difference is that GetBalance sums up all unspent TxOuts paying to the +// wallet, while this sums up both spent and unspent TxOuts paying to the +// wallet, and then subtracts the values of TxIns spending from the wallet. This +// also has fewer restrictions on which unconfirmed transactions are considered +// trusted. +CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + for (const auto& entry : mapWallet) { + const CWalletTx& wtx = entry.second; + const int depth = wtx.GetDepthInMainChain(); + if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.GetBlocksToMaturity() > 0) { + continue; + } + + // Loop through tx outputs and add incoming payments. For outgoing txs, + // treat change outputs specially, as part of the amount debited. + CAmount debit = wtx.GetDebit(filter); + const bool outgoing = debit > 0; + for (const CTxOut& out : wtx.tx->vout) { + if (outgoing && IsChange(out)) { + debit -= out.nValue; + } else if (IsMine(out) & filter && depth >= minDepth && (!account || *account == GetAccountName(out.scriptPubKey))) { + balance += out.nValue; + } + } + + // For outgoing txs, subtract amount debited. + if (outgoing && (!account || *account == wtx.strFromAccount)) { + balance -= debit; + } + } + + if (account) { + balance += CWalletDB(*dbw).GetAccountCreditDebit(*account); + } + + return balance; +} + +void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const { vCoins.clear(); { LOCK2(cs_main, cs_wallet); + + CAmount nTotal = 0; + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const uint256& wtxid = it->first; @@ -2017,21 +2043,52 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const continue; } + if (nDepth < nMinDepth || nDepth > nMaxDepth) + continue; + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + continue; + + if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint((*it).first, i))) + continue; + + if (IsLockedCoin((*it).first, i)) + continue; + + if (IsSpent(wtxid, i)) + continue; + isminetype mine = IsMine(pcoin->tx->vout[i]); - if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue) && - (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) - vCoins.push_back(COutput(pcoin, i, nDepth, - ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || - (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO), - (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO, safeTx)); + + if (mine == ISMINE_NO) { + continue; + } + + bool fSpendableIn = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO); + bool fSolvableIn = (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO; + + vCoins.push_back(COutput(pcoin, i, nDepth, fSpendableIn, fSolvableIn, safeTx)); + + // Checks the sum amount of all UTXO's. + if (nMinimumSumAmount != MAX_MONEY) { + nTotal += pcoin->tx->vout[i].nValue; + + if (nTotal >= nMinimumSumAmount) { + return; + } + } + + // Checks the maximum number of UTXO's. + if (nMaximumCount > 0 && vCoins.size() >= nMaximumCount) { + return; + } } } } } -static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > >& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, +static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { std::vector<char> vfIncluded; @@ -2056,9 +2113,9 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair //that the rng is fast. We do not use a constant random sequence, //because there may be some privacy improvement by making //the selection random. - if (nPass == 0 ? insecure_rand.rand32()&1 : !vfIncluded[i]) + if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i]) { - nTotal += vValue[i].first; + nTotal += vValue[i].txout.nValue; vfIncluded[i] = true; if (nTotal >= nTargetValue) { @@ -2068,7 +2125,7 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair nBest = nTotal; vfBest = vfIncluded; } - nTotal -= vValue[i].first; + nTotal -= vValue[i].txout.nValue; vfIncluded[i] = false; } } @@ -2078,16 +2135,14 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair } bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMine, const int nConfTheirs, const uint64_t nMaxAncestors, std::vector<COutput> vCoins, - std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const + std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const { setCoinsRet.clear(); nValueRet = 0; // List of values less than target - std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > coinLowestLarger; - coinLowestLarger.first = std::numeric_limits<CAmount>::max(); - coinLowestLarger.second.first = NULL; - std::vector<std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > > vValue; + boost::optional<CInputCoin> coinLowestLarger; + std::vector<CInputCoin> vValue; CAmount nTotalLower = 0; random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); @@ -2106,22 +2161,21 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin continue; int i = output.i; - CAmount n = pcoin->tx->vout[i].nValue; - std::pair<CAmount,std::pair<const CWalletTx*,unsigned int> > coin = std::make_pair(n,std::make_pair(pcoin, i)); + CInputCoin coin = CInputCoin(pcoin, i); - if (n == nTargetValue) + if (coin.txout.nValue == nTargetValue) { - setCoinsRet.insert(coin.second); - nValueRet += coin.first; + setCoinsRet.insert(coin); + nValueRet += coin.txout.nValue; return true; } - else if (n < nTargetValue + MIN_CHANGE) + else if (coin.txout.nValue < nTargetValue + MIN_CHANGE) { vValue.push_back(coin); - nTotalLower += n; + nTotalLower += coin.txout.nValue; } - else if (n < coinLowestLarger.first) + else if (!coinLowestLarger || coin.txout.nValue < coinLowestLarger->txout.nValue) { coinLowestLarger = coin; } @@ -2131,18 +2185,18 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin { for (unsigned int i = 0; i < vValue.size(); ++i) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(vValue[i]); + nValueRet += vValue[i].txout.nValue; } return true; } if (nTotalLower < nTargetValue) { - if (coinLowestLarger.second.first == NULL) + if (!coinLowestLarger) return false; - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; return true; } @@ -2158,25 +2212,25 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin // If we have a bigger coin and (either the stochastic approximation didn't find a good solution, // or the next bigger coin is closer), return the bigger coin - if (coinLowestLarger.second.first && - ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger.first <= nBest)) + if (coinLowestLarger && + ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger->txout.nValue <= nBest)) { - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; } else { for (unsigned int i = 0; i < vValue.size(); i++) if (vfBest[i]) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(vValue[i]); + nValueRet += vValue[i].txout.nValue; } if (LogAcceptCategory(BCLog::SELECTCOINS)) { LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); for (unsigned int i = 0; i < vValue.size(); i++) { if (vfBest[i]) { - LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(vValue[i].first)); + LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(vValue[i].txout.nValue)); } } LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest)); @@ -2186,7 +2240,7 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin return true; } -bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const +bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const { std::vector<COutput> vCoins(vAvailableCoins); @@ -2198,13 +2252,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm if (!out.fSpendable) continue; nValueRet += out.tx->tx->vout[out.i].nValue; - setCoinsRet.insert(std::make_pair(out.tx, out.i)); + setCoinsRet.insert(CInputCoin(out.tx, out.i)); } return (nValueRet >= nTargetValue); } // calculate value from preset inputs and store them - std::set<std::pair<const CWalletTx*, uint32_t> > setPresetCoins; + std::set<CInputCoin> setPresetCoins; CAmount nValueFromPresetInputs = 0; std::vector<COutPoint> vPresetInputs; @@ -2220,7 +2274,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm if (pcoin->tx->vout.size() <= outpoint.n) return false; nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(std::make_pair(pcoin, outpoint.n)); + setPresetCoins.insert(CInputCoin(pcoin, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } @@ -2228,7 +2282,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // remove preset inputs from vCoins for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coinControl && coinControl->HasSelected();) { - if (setPresetCoins.count(std::make_pair(it->tx, it->i))) + if (setPresetCoins.count(CInputCoin(it->tx, it->i))) it = vCoins.erase(it); else ++it; @@ -2255,6 +2309,30 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm return res; } +bool CWallet::SignTransaction(CMutableTransaction &tx) +{ + AssertLockHeld(cs_wallet); // mapWallet + + // sign the new tx + CTransaction txNewConst(tx); + int nIn = 0; + for (auto& input : tx.vin) { + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(input.prevout.hash); + if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { + return false; + } + const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; + const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; + SignatureData sigdata; + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { + return false; + } + UpdateTransaction(tx, nIn, sigdata); + nIn++; + } + return true; +} + bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, bool keepReserveKey, const CTxDestination& destChange) { std::vector<CRecipient> vecSend; @@ -2372,7 +2450,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT assert(txNew.nLockTime < LOCKTIME_THRESHOLD); { - std::set<std::pair<const CWalletTx*,unsigned int> > setCoins; + std::set<CInputCoin> setCoins; LOCK2(cs_main, cs_wallet); { std::vector<COutput> vAvailableCoins; @@ -2407,7 +2485,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } } - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2472,16 +2550,16 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // We do not move dust-change to fees, because the sender would end up paying more than requested. // This would be against the purpose of the all-inclusive feature. // So instead we raise the change and deduct from the recipient. - if (nSubtractFeeFromAmount > 0 && newTxOut.IsDust(dustRelayFee)) + if (nSubtractFeeFromAmount > 0 && IsDust(newTxOut, ::dustRelayFee)) { - CAmount nDust = newTxOut.GetDustThreshold(dustRelayFee) - newTxOut.nValue; + CAmount nDust = GetDustThreshold(newTxOut, ::dustRelayFee) - newTxOut.nValue; newTxOut.nValue += nDust; // raise change until no more dust for (unsigned int i = 0; i < vecSend.size(); i++) // subtract from first recipient { if (vecSend[i].fSubtractFeeFromAmount) { txNew.vout[i].nValue -= nDust; - if (txNew.vout[i].IsDust(dustRelayFee)) + if (IsDust(txNew.vout[i], ::dustRelayFee)) { strFailReason = _("The transaction amount is too small to send after the fee has been deducted"); return false; @@ -2493,7 +2571,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // Never create dust outputs; if we would, just // add the dust to the fee. - if (newTxOut.IsDust(dustRelayFee)) + if (IsDust(newTxOut, ::dustRelayFee)) { nChangePosInOut = -1; nFeeRet += nChange; @@ -2515,9 +2593,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT std::vector<CTxOut>::iterator position = txNew.vout.begin()+nChangePosInOut; txNew.vout.insert(position, newTxOut); } - } - else + } else { reservekey.ReturnKey(); + nChangePosInOut = -1; + } // Fill vin // @@ -2531,7 +2610,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // behavior." bool rbf = coinControl ? coinControl->signalRbf : fWalletRbf; for (const auto& coin : setCoins) - txNew.vin.push_back(CTxIn(coin.first->GetHash(),coin.second,CScript(), + txNew.vin.push_back(CTxIn(coin.outpoint,CScript(), std::numeric_limits<unsigned int>::max() - (rbf ? 2 : 1))); // Fill in dummy signatures for fee calculation. @@ -2555,7 +2634,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT if (coinControl && coinControl->nConfirmTarget > 0) currentConfirmationTarget = coinControl->nConfirmTarget; - CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } @@ -2614,10 +2693,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT int nIn = 0; for (const auto& coin : setCoins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->tx->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata)) + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed"); return false; @@ -2701,13 +2780,13 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon } void CWallet::ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return walletdb.ListAccountCreditDebit(strAccount, entries); } bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return AddAccountingEntry(acentry, &walletdb); } @@ -2729,19 +2808,14 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreUserSetFee) { // payTxFee is the user-set global for desired feerate - return GetMinimumFee(nTxBytes, nConfirmTarget, pool, payTxFee.GetFee(nTxBytes)); -} - -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee) -{ - CAmount nFeeNeeded = targetFee; + CAmount nFeeNeeded = payTxFee.GetFee(nTxBytes); // User didn't set: use -txconfirmtarget to estimate... - if (nFeeNeeded == 0) { + if (nFeeNeeded == 0 || ignoreUserSetFee) { int estimateFoundTarget = nConfirmTarget; - nFeeNeeded = pool.estimateSmartFee(nConfirmTarget, &estimateFoundTarget).GetFee(nTxBytes); + nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = fallbackFee.GetFee(nTxBytes); @@ -2759,13 +2833,11 @@ CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarge DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - if (!fFileBacked) - return DB_LOAD_OK; fFirstRunRet = false; - DBErrors nLoadWalletRet = CWalletDB(strWalletFile,"cr+").LoadWallet(this); + DBErrors nLoadWalletRet = CWalletDB(*dbw,"cr+").LoadWallet(this); if (nLoadWalletRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2786,17 +2858,15 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - if (!fFileBacked) - return DB_LOAD_OK; AssertLockHeld(cs_wallet); // mapWallet vchDefaultKey = CPubKey(); - DBErrors nZapSelectTxRet = CWalletDB(strWalletFile,"cr+").ZapSelectTx(vHashIn, vHashOut); + DBErrors nZapSelectTxRet = CWalletDB(*dbw,"cr+").ZapSelectTx(vHashIn, vHashOut); for (uint256 hash : vHashOut) mapWallet.erase(hash); if (nZapSelectTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { setKeyPool.clear(); // Note: can't top-up keypool here, because wallet is locked. @@ -2816,13 +2886,11 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { - if (!fFileBacked) - return DB_LOAD_OK; vchDefaultKey = CPubKey(); - DBErrors nZapWalletTxRet = CWalletDB(strWalletFile,"cr+").ZapWalletTx(vWtx); + DBErrors nZapWalletTxRet = CWalletDB(*dbw,"cr+").ZapWalletTx(vWtx); if (nZapWalletTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2852,11 +2920,9 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s } NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); - if (!fFileBacked) + if (!strPurpose.empty() && !CWalletDB(*dbw).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) return false; - if (!strPurpose.empty() && !CWalletDB(strWalletFile).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) - return false; - return CWalletDB(strWalletFile).WriteName(CBitcoinAddress(address).ToString(), strName); + return CWalletDB(*dbw).WriteName(CBitcoinAddress(address).ToString(), strName); } bool CWallet::DelAddressBook(const CTxDestination& address) @@ -2864,33 +2930,40 @@ bool CWallet::DelAddressBook(const CTxDestination& address) { LOCK(cs_wallet); // mapAddressBook - if(fFileBacked) + // Delete destdata tuples associated with address + std::string strAddress = CBitcoinAddress(address).ToString(); + BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) { - // Delete destdata tuples associated with address - std::string strAddress = CBitcoinAddress(address).ToString(); - BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) - { - CWalletDB(strWalletFile).EraseDestData(strAddress, item.first); - } + CWalletDB(*dbw).EraseDestData(strAddress, item.first); } mapAddressBook.erase(address); } NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); - if (!fFileBacked) - return false; - CWalletDB(strWalletFile).ErasePurpose(CBitcoinAddress(address).ToString()); - return CWalletDB(strWalletFile).EraseName(CBitcoinAddress(address).ToString()); + CWalletDB(*dbw).ErasePurpose(CBitcoinAddress(address).ToString()); + return CWalletDB(*dbw).EraseName(CBitcoinAddress(address).ToString()); } -bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +const std::string& CWallet::GetAccountName(const CScript& scriptPubKey) const { - if (fFileBacked) - { - if (!CWalletDB(strWalletFile).WriteDefaultKey(vchPubKey)) - return false; + CTxDestination address; + if (ExtractDestination(scriptPubKey, address) && !scriptPubKey.IsUnspendable()) { + auto mi = mapAddressBook.find(address); + if (mi != mapAddressBook.end()) { + return mi->second.name; + } } + // A scriptPubKey that doesn't have an entry in the address book is + // associated with the default account (""). + const static std::string DEFAULT_ACCOUNT_NAME; + return DEFAULT_ACCOUNT_NAME; +} + +bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +{ + if (!CWalletDB(*dbw).WriteDefaultKey(vchPubKey)) + return false; vchDefaultKey = vchPubKey; return true; } @@ -2903,7 +2976,7 @@ bool CWallet::NewKeyPool() { { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); BOOST_FOREACH(int64_t nIndex, setKeyPool) walletdb.ErasePool(nIndex); setKeyPool.clear(); @@ -2924,7 +2997,7 @@ size_t CWallet::KeypoolCountExternalKeys() if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) return setKeyPool.size(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // count amount of external keys size_t amountE = 0; @@ -2967,7 +3040,7 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) missingInternal = 0; } bool internal = false; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); for (int64_t i = missingInternal + missingExternal; i--;) { int64_t nEnd = 1; @@ -2998,7 +3071,7 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool int if(setKeyPool.empty()) return; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // try to find a key that matches the internal/external filter for(const int64_t& id : setKeyPool) @@ -3024,11 +3097,8 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool int void CWallet::KeepKey(int64_t nIndex) { // Remove from key pool - if (fFileBacked) - { - CWalletDB walletdb(strWalletFile); - walletdb.ErasePool(nIndex); - } + CWalletDB walletdb(*dbw); + walletdb.ErasePool(nIndex); LogPrintf("keypool keep %d\n", nIndex); } @@ -3070,7 +3140,7 @@ int64_t CWallet::GetOldestKeyPoolTime() return GetTime(); CKeyPool keypool; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) { @@ -3236,37 +3306,6 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() return ret; } -CAmount CWallet::GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CWalletDB walletdb(strWalletFile); - return GetAccountBalance(walletdb, strAccount, nMinDepth, filter); -} - -CAmount CWallet::GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CAmount nBalance = 0; - - // Tally wallet transactions - for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount nReceived, nSent, nFee; - wtx.GetAccountAmounts(strAccount, nReceived, nSent, nFee, filter); - - if (nReceived != 0 && wtx.GetDepthInMainChain() >= nMinDepth) - nBalance += nReceived; - nBalance -= nSent + nFee; - } - - // Tally internal accounting entries - nBalance += walletdb.GetAccountCreditDebit(strAccount); - - return nBalance; -} - std::set<CTxDestination> CWallet::GetAccountAddresses(const std::string& strAccount) const { LOCK(cs_wallet); @@ -3318,7 +3357,7 @@ void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const { setAddress.clear(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); LOCK2(cs_main, cs_wallet); BOOST_FOREACH(const int64_t& id, setKeyPool) @@ -3334,20 +3373,9 @@ void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const } } -void CWallet::UpdatedTransaction(const uint256 &hashTx) +void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) { - { - LOCK(cs_wallet); - // Only notify UI if this transaction is in this wallet - std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(hashTx); - if (mi != mapWallet.end()) - NotifyTransactionChanged(this, hashTx, CT_UPDATED); - } -} - -void CWallet::GetScriptForMining(boost::shared_ptr<CReserveScript> &script) -{ - boost::shared_ptr<CReserveKey> rKey(new CReserveKey(this)); + std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); CPubKey pubkey; if (!rKey->GetReservedKey(pubkey)) return; @@ -3551,18 +3579,14 @@ bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, co return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); + return CWalletDB(*dbw).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); } bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).EraseDestData(CBitcoinAddress(dest).ToString(), key); + return CWalletDB(*dbw).EraseDestData(CBitcoinAddress(dest).ToString(), key); } bool CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -3632,7 +3656,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (GetBoolArg("-zapwallettxes", false)) { uiInterface.InitMessage(_("Zapping all transactions from wallet...")); - CWallet *tempWallet = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *tempWallet = new CWallet(std::move(dbw)); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -3647,7 +3672,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) int64_t nStart = GetTimeMillis(); bool fFirstRun = true; - CWallet *walletInstance = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *walletInstance = new CWallet(std::move(dbw)); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DB_LOAD_OK) { @@ -3738,7 +3764,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) CBlockIndex *pindexRescan = chainActive.Genesis(); if (!GetBoolArg("-rescan", false)) { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); CBlockLocator locator; if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); @@ -3771,7 +3797,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Restore wallet transaction metadata after -zapwallettxes=1 if (GetBoolArg("-zapwallettxes", false) && GetArg("-zapwallettxes", "1") != "2") { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); BOOST_FOREACH(const CWalletTx& wtxOld, vWtx) { @@ -3815,7 +3841,7 @@ bool CWallet::InitLoadWallet() std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - if (walletFile.find_first_of("/\\") != std::string::npos) { + if (boost::filesystem::path(walletFile).filename() != walletFile) { return InitError(_("-wallet parameter must only specify a filename (not a path)")); } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { return InitError(_("Invalid characters in -wallet filename")); @@ -3931,38 +3957,7 @@ bool CWallet::ParameterInteraction() bool CWallet::BackupWallet(const std::string& strDest) { - if (!fFileBacked) - return false; - while (true) - { - { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strWalletFile) || bitdb.mapFileUseCount[strWalletFile] == 0) - { - // Flush log data to the dat file - bitdb.CloseDb(strWalletFile); - bitdb.CheckpointLSN(strWalletFile); - bitdb.mapFileUseCount.erase(strWalletFile); - - // Copy wallet file - fs::path pathSrc = GetDataDir() / strWalletFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) - pathDest /= strWalletFile; - - try { - fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strWalletFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strWalletFile, pathDest.string(), e.what()); - return false; - } - } - } - MilliSleep(100); - } - return false; + return dbw->Backup(strDest); } CKeyPool::CKeyPool() diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index ccede60097..69f51b3f64 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_WALLET_H #include "amount.h" +#include "policy/feerate.h" #include "streams.h" #include "tinyformat.h" #include "ui_interface.h" @@ -28,8 +29,6 @@ #include <utility> #include <vector> -#include <boost/shared_ptr.hpp> - extern CWallet* pwalletMain; /** @@ -75,6 +74,7 @@ class CReserveKey; class CScript; class CScheduler; class CTxMemPool; +class CBlockPolicyEstimator; class CWalletTx; /** (client) version numbers for particular wallet features */ @@ -452,9 +452,6 @@ public: void GetAmounts(std::list<COutputEntry>& listReceived, std::list<COutputEntry>& listSent, CAmount& nFee, std::string& strSentAccount, const isminefilter& filter) const; - void GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const; - bool IsFromMe(const isminefilter& filter) const { return (GetDebit(filter) > 0); @@ -475,7 +472,34 @@ public: }; +class CInputCoin { +public: + CInputCoin(const CWalletTx* walletTx, unsigned int i) + { + if (!walletTx) + throw std::invalid_argument("walletTx should not be null"); + if (i >= walletTx->tx->vout.size()) + throw std::out_of_range("The output index is out of range"); + + outpoint = COutPoint(walletTx->GetHash(), i); + txout = walletTx->tx->vout[i]; + } + + COutPoint outpoint; + CTxOut txout; + + bool operator<(const CInputCoin& rhs) const { + return outpoint < rhs.outpoint; + } + + bool operator!=(const CInputCoin& rhs) const { + return outpoint != rhs.outpoint; + } + bool operator==(const CInputCoin& rhs) const { + return outpoint == rhs.outpoint; + } +}; class COutput { @@ -626,13 +650,15 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface { private: static std::atomic<bool> fFlushScheduled; + std::atomic<bool> fAbortRescan; + std::atomic<bool> fScanningWallet; /** * Select a set of coins such that nValueRet >= nTargetValue and at least * all coins from coinControl are selected; Never select unconfirmed coins * if they are not ours */ - bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; + bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; CWalletDB *pwalletdbEncryption; @@ -661,14 +687,16 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>); + /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected. + * Should be called with pindexBlock and posInBlock if this is for a transaction that is included in a block. */ + void SyncTransaction(const CTransactionRef& tx, const CBlockIndex *pindex = NULL, int posInBlock = 0); + /* the HD chain data model (external chain counters) */ CHDChain hdChain; /* HD derive new child key (on internal or external chain) */ void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal = false); - bool fFileBacked; - std::set<int64_t> setKeyPool; int64_t nTimeFirstKey; @@ -684,17 +712,33 @@ private: */ bool AddWatchOnly(const CScript& dest) override; + std::unique_ptr<CWalletDBWrapper> dbw; + public: /* * Main wallet lock. - * This lock protects all the fields added by CWallet - * except for: - * fFileBacked (immutable after instantiation) - * strWalletFile (immutable after instantiation) + * This lock protects all the fields added by CWallet. */ mutable CCriticalSection cs_wallet; - const std::string strWalletFile; + /** Get database handle used by this wallet. Ideally this function would + * not be necessary. + */ + CWalletDBWrapper& GetDBHandle() + { + return *dbw; + } + + /** Get a name for this wallet for logging/debugging purposes. + */ + std::string GetName() const + { + if (dbw) { + return dbw->GetName(); + } else { + return "dummy"; + } + } void LoadKeyPool(int nIndex, const CKeyPool &keypool) { @@ -716,15 +760,16 @@ public: MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID; - CWallet() + // Create wallet with dummy database handle + CWallet(): dbw(new CWalletDBWrapper()) { SetNull(); } - CWallet(const std::string& strWalletFileIn) : strWalletFile(strWalletFileIn) + // Create wallet with passed-in database handle + CWallet(std::unique_ptr<CWalletDBWrapper> dbw_in) : dbw(std::move(dbw_in)) { SetNull(); - fFileBacked = true; } ~CWallet() @@ -737,7 +782,6 @@ public: { nWalletVersion = FEATURE_BASE; nWalletMaxVersion = FEATURE_BASE; - fFileBacked = false; nMasterKeyMaxID = 0; pwalletdbEncryption = NULL; nOrderPosNext = 0; @@ -746,6 +790,8 @@ public: nTimeFirstKey = 0; fBroadcastTransactions = false; nRelockTime = 0; + fAbortRescan = false; + fScanningWallet = false; } std::map<uint256, CWalletTx> mapWallet; @@ -772,7 +818,7 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, bool fIncludeZeroValue=false) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; /** * Shuffle and select coins until nTargetValue is reached while avoiding @@ -780,7 +826,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const; + bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; @@ -790,6 +836,13 @@ public: void UnlockAllCoins(); void ListLockedCoins(std::vector<COutPoint>& vOutpts); + /* + * Rescan abort properties + */ + void AbortRescan() { fAbortRescan = true; } + bool IsAbortingRescan() { return fAbortRescan; } + bool IsScanning() { return fScanningWallet; } + /** * keystore implementation * Generate a new key @@ -849,8 +902,10 @@ public: void MarkDirty(); bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); bool LoadToWallet(const CWalletTx& wtxIn); - void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock) override; - bool AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); + void TransactionAddedToMempool(const CTransactionRef& tx) override; + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; @@ -861,12 +916,14 @@ public: CAmount GetWatchOnlyBalance() const; CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; + CAmount GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const; /** * Insert additional inputs into the transaction by * calling CreateTransaction(); */ bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, bool keepReserveKey = true, const CTxDestination& destChange = CNoDestination()); + bool SignTransaction(CMutableTransaction& tx); /** * Create a new transaction paying the recipients with a set of coins @@ -881,7 +938,7 @@ public: bool AddAccountingEntry(const CAccountingEntry&); bool AddAccountingEntry(const CAccountingEntry&, CWalletDB *pwalletdb); template <typename ContainerType> - bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins); + bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const; static CFeeRate minTxFee; static CFeeRate fallbackFee; @@ -889,12 +946,7 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool); - /** - * Estimate the minimum fee considering required fee and targetFee or if 0 - * then fee estimation for nConfirmTarget - */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreUserSetFee = false); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -914,8 +966,6 @@ public: std::set< std::set<CTxDestination> > GetAddressGroupings(); std::map<CTxDestination, CAmount> GetAddressBalances(); - CAmount GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter); - CAmount GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter); std::set<CTxDestination> GetAccountAddresses(const std::string& strAccount) const; isminetype IsMine(const CTxIn& txin) const; @@ -946,7 +996,7 @@ public: bool DelAddressBook(const CTxDestination& address); - void UpdatedTransaction(const uint256 &hashTx) override; + const std::string& GetAccountName(const CScript& scriptPubKey) const; void Inventory(const uint256 &hash) override { @@ -958,12 +1008,7 @@ public: } } - void GetScriptForMining(boost::shared_ptr<CReserveScript> &script) override; - void ResetRequestCount(const uint256 &hash) override - { - LOCK(cs_wallet); - mapRequestCount[hash] = 0; - }; + void GetScriptForMining(std::shared_ptr<CReserveScript> &script) override; unsigned int GetKeyPoolSize() { @@ -1056,9 +1101,10 @@ public: CPubKey GenerateNewHDMasterKey(); /* Set the current HD master key (will reset the chain child index counters) - If possibleOldChain is provided, the parameters from the old chain (version) - will be preserved. */ - bool SetHDMasterKey(const CPubKey& key, CHDChain *possibleOldChain = nullptr); + Sets the master key's version based on the current wallet version (so the + caller must ensure the current wallet version is correct before calling + this function). */ + bool SetHDMasterKey(const CPubKey& key); }; /** A key allocated from the key pool. */ @@ -1125,13 +1171,13 @@ public: // ContainerType is meant to hold pair<CWalletTx *, int>, and be iterable // so that each entry corresponds to each vIn, in order. template <typename ContainerType> -bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) +bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const { // Fill in dummy signatures for fee calculation. int nIn = 0; for (const auto& coin : coins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; if (!ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata)) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 1beeaa042e..342c797dd3 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -33,7 +33,7 @@ static std::atomic<unsigned int> nWalletDBUpdateCounter; bool CWalletDB::WriteName(const std::string& strAddress, const std::string& strName) { nWalletDBUpdateCounter++; - return Write(make_pair(std::string("name"), strAddress), strName); + return batch.Write(std::make_pair(std::string("name"), strAddress), strName); } bool CWalletDB::EraseName(const std::string& strAddress) @@ -41,38 +41,38 @@ bool CWalletDB::EraseName(const std::string& strAddress) // This should only be used for sending addresses, never for receiving addresses, // receiving addresses must always have an address book entry if they're not change return. nWalletDBUpdateCounter++; - return Erase(make_pair(std::string("name"), strAddress)); + return batch.Erase(std::make_pair(std::string("name"), strAddress)); } bool CWalletDB::WritePurpose(const std::string& strAddress, const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Write(make_pair(std::string("purpose"), strAddress), strPurpose); + return batch.Write(std::make_pair(std::string("purpose"), strAddress), strPurpose); } bool CWalletDB::ErasePurpose(const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Erase(make_pair(std::string("purpose"), strPurpose)); + return batch.Erase(std::make_pair(std::string("purpose"), strPurpose)); } bool CWalletDB::WriteTx(const CWalletTx& wtx) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); + return batch.Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); } bool CWalletDB::EraseTx(uint256 hash) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("tx"), hash)); + return batch.Erase(std::make_pair(std::string("tx"), hash)); } bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) return false; @@ -82,7 +82,7 @@ bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, c vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); vchKey.insert(vchKey.end(), vchPrivKey.begin(), vchPrivKey.end()); - return Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); + return batch.Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); } bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, @@ -92,16 +92,16 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, const bool fEraseUnencryptedKey = true; nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) return false; - if (!Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) + if (!batch.Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) return false; if (fEraseUnencryptedKey) { - Erase(std::make_pair(std::string("key"), vchPubKey)); - Erase(std::make_pair(std::string("wkey"), vchPubKey)); + batch.Erase(std::make_pair(std::string("key"), vchPubKey)); + batch.Erase(std::make_pair(std::string("wkey"), vchPubKey)); } return true; } @@ -109,92 +109,92 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); + return batch.Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); } bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); + return batch.Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); } bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) + if (!batch.Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) return false; - return Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); + return batch.Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); } bool CWalletDB::EraseWatchOnly(const CScript &dest) { nWalletDBUpdateCounter++; - if (!Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) + if (!batch.Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) return false; - return Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); + return batch.Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); } bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) { nWalletDBUpdateCounter++; - Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan - return Write(std::string("bestblock_nomerkle"), locator); + batch.Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + return batch.Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) { - if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; - return Read(std::string("bestblock_nomerkle"), locator); + if (batch.Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; + return batch.Read(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::WriteOrderPosNext(int64_t nOrderPosNext) { nWalletDBUpdateCounter++; - return Write(std::string("orderposnext"), nOrderPosNext); + return batch.Write(std::string("orderposnext"), nOrderPosNext); } bool CWalletDB::WriteDefaultKey(const CPubKey& vchPubKey) { nWalletDBUpdateCounter++; - return Write(std::string("defaultkey"), vchPubKey); + return batch.Write(std::string("defaultkey"), vchPubKey); } bool CWalletDB::ReadPool(int64_t nPool, CKeyPool& keypool) { - return Read(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Read(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::WritePool(int64_t nPool, const CKeyPool& keypool) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Write(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::ErasePool(int64_t nPool) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("pool"), nPool)); + return batch.Erase(std::make_pair(std::string("pool"), nPool)); } bool CWalletDB::WriteMinVersion(int nVersion) { - return Write(std::string("minversion"), nVersion); + return batch.Write(std::string("minversion"), nVersion); } bool CWalletDB::ReadAccount(const std::string& strAccount, CAccount& account) { account.SetNull(); - return Read(make_pair(std::string("acc"), strAccount), account); + return batch.Read(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccount(const std::string& strAccount, const CAccount& account) { - return Write(make_pair(std::string("acc"), strAccount), account); + return batch.Write(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry) { - return Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); + return batch.Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); } bool CWalletDB::WriteAccountingEntry_Backend(const CAccountingEntry& acentry) @@ -218,7 +218,7 @@ void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list< { bool fAllAccounts = (strAccount == "*"); - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) throw std::runtime_error(std::string(__func__) + ": cannot create DB cursor"); bool setRange = true; @@ -229,7 +229,7 @@ void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list< if (setRange) ssKey << std::make_pair(std::string("acentry"), std::make_pair((fAllAccounts ? std::string("") : strAccount), uint64_t(0))); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue, setRange); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue, setRange); setRange = false; if (ret == DB_NOTFOUND) break; @@ -560,7 +560,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (Read((std::string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; @@ -568,7 +568,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -580,7 +580,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -664,14 +664,14 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal try { int nMinVersion = 0; - if (Read((std::string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -683,7 +683,7 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -797,9 +797,9 @@ void MaybeCompactWalletDB() if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) { - const std::string& strFile = pwalletMain->strWalletFile; - if (CDB::PeriodicFlush(strFile)) + if (CDB::PeriodicFlush(pwalletMain->GetDBHandle())) { nLastFlushed = CWalletDB::GetUpdateCounter(); + } } fOneThread = false; } @@ -855,20 +855,20 @@ bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); + return batch.Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); } bool CWalletDB::EraseDestData(const std::string &address, const std::string &key) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); + return batch.Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); } bool CWalletDB::WriteHDChain(const CHDChain& chain) { nWalletDBUpdateCounter++; - return Write(std::string("hdchain"), chain); + return batch.Write(std::string("hdchain"), chain); } void CWalletDB::IncrementUpdateCounter() @@ -880,3 +880,28 @@ unsigned int CWalletDB::GetUpdateCounter() { return nWalletDBUpdateCounter; } + +bool CWalletDB::TxnBegin() +{ + return batch.TxnBegin(); +} + +bool CWalletDB::TxnCommit() +{ + return batch.TxnCommit(); +} + +bool CWalletDB::TxnAbort() +{ + return batch.TxnAbort(); +} + +bool CWalletDB::ReadVersion(int& nVersion) +{ + return batch.ReadVersion(nVersion); +} + +bool CWalletDB::WriteVersion(int nVersion) +{ + return batch.WriteVersion(nVersion); +} diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index b94f341b2e..cd9fe279c5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -17,6 +17,21 @@ #include <utility> #include <vector> +/** + * Overview of wallet database classes: + * + * - CDBEnv is an environment in which the database exists (has no analog in dbwrapper.h) + * - CWalletDBWrapper represents a wallet database (similar to CDBWrapper in dbwrapper.h) + * - CDB is a low-level database transaction (similar to CDBBatch in dbwrapper.h) + * - CWalletDB is a modifier object for the wallet, and encapsulates a database + * transaction as well as methods to act on the database (no analog in + * dbwrapper.h) + * + * The latter two are named confusingly, in contrast to what the names CDB + * and CWalletDB suggest they are transient transaction objects and don't + * represent the database itself. + */ + static const bool DEFAULT_FLUSHWALLET = true; class CAccount; @@ -118,11 +133,16 @@ public: } }; -/** Access to the wallet database */ -class CWalletDB : public CDB +/** Access to the wallet database. + * This should really be named CWalletDBBatch, as it represents a single transaction at the + * database. It will be committed when the object goes out of scope. + * Optionally (on by default) it will flush to disk as well. + */ +class CWalletDB { public: - CWalletDB(const std::string& strFilename, const char* pszMode = "r+", bool _fFlushOnClose = true) : CDB(strFilename, pszMode, _fFlushOnClose) + CWalletDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool _fFlushOnClose = true) : + batch(dbw, pszMode, _fFlushOnClose) { } @@ -194,7 +214,20 @@ public: static void IncrementUpdateCounter(); static unsigned int GetUpdateCounter(); + + //! Begin a new transaction + bool TxnBegin(); + //! Commit current transaction + bool TxnCommit(); + //! Abort current transaction + bool TxnAbort(); + //! Read wallet version + bool ReadVersion(int& nVersion); + //! Write wallet version + bool WriteVersion(int nVersion); private: + CDB batch; + CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); }; diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index fac2a3c57a..c063898056 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -144,8 +144,12 @@ void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindexNew, co } } -void CZMQNotificationInterface::SyncTransaction(const CTransaction& tx, const CBlockIndex* pindex, int posInBlock) +void CZMQNotificationInterface::TransactionAddedToMempool(const CTransactionRef& ptx) { + // Used by BlockConnected and BlockDisconnected as well, because they're + // all the same external callback. + const CTransaction& tx = *ptx; + for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) { CZMQAbstractNotifier *notifier = *i; @@ -160,3 +164,19 @@ void CZMQNotificationInterface::SyncTransaction(const CTransaction& tx, const CB } } } + +void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) +{ + for (const CTransactionRef& ptx : pblock->vtx) { + // Do a normal notify for each transaction added in the block + TransactionAddedToMempool(ptx); + } +} + +void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) +{ + for (const CTransactionRef& ptx : pblock->vtx) { + // Do a normal notify for each transaction removed in block disconnection + TransactionAddedToMempool(ptx); + } +} diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index beabb78da6..eec6f7bc64 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -8,6 +8,7 @@ #include "validationinterface.h" #include <string> #include <map> +#include <list> class CBlockIndex; class CZMQAbstractNotifier; @@ -24,8 +25,10 @@ protected: void Shutdown(); // CValidationInterface - void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, int posInBlock); - void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload); + void TransactionAddedToMempool(const CTransactionRef& tx) override; + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; private: CZMQNotificationInterface(); diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index b2963e9bde..700c39f66e 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "chain.h" #include "chainparams.h" #include "streams.h" #include "zmqpublishnotifier.h" |