diff options
Diffstat (limited to 'src')
113 files changed, 4345 insertions, 2444 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index cb88171348..ae2eb29c94 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -95,6 +95,7 @@ BITCOIN_CORE_H = \ compat/sanity.h \ compressor.h \ consensus/consensus.h \ + consensus/tx_verify.h \ core_io.h \ core_memusage.h \ cuckoocache.h \ @@ -116,6 +117,7 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ noui.h \ + policy/feerate.h \ policy/fees.h \ policy/policy.h \ policy/rbf.h \ @@ -178,12 +180,13 @@ libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ - addrman.cpp \ addrdb.cpp \ + addrman.cpp \ bloom.cpp \ blockencodings.cpp \ chain.cpp \ checkpoints.cpp \ + consensus/tx_verify.cpp \ httprpc.cpp \ httpserver.cpp \ init.cpp \ @@ -301,7 +304,6 @@ libbitcoin_consensus_a_SOURCES = \ libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ - amount.cpp \ base58.cpp \ chainparams.cpp \ coins.cpp \ @@ -312,6 +314,7 @@ libbitcoin_common_a_SOURCES = \ keystore.cpp \ netaddress.cpp \ netbase.cpp \ + policy/feerate.cpp \ protocol.cpp \ scheduler.cpp \ script/sign.cpp \ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 948e13a9e1..391b9ebdf6 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -46,7 +46,8 @@ qt_test_test_bitcoin_qt_SOURCES = \ if ENABLE_WALLET qt_test_test_bitcoin_qt_SOURCES += \ qt/test/paymentservertests.cpp \ - qt/test/wallettests.cpp + qt/test/wallettests.cpp \ + wallet/test/wallet_test_fixture.cpp endif nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 10cb7e775a..ee1c11ff1f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -78,6 +78,7 @@ BITCOIN_TESTS =\ test/testutil.cpp \ test/testutil.h \ test/timedata_tests.cpp \ + test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ diff --git a/src/amount.h b/src/amount.h index 93060f7193..2bd367cba2 100644 --- a/src/amount.h +++ b/src/amount.h @@ -6,10 +6,7 @@ #ifndef BITCOIN_AMOUNT_H #define BITCOIN_AMOUNT_H -#include "serialize.h" - -#include <stdlib.h> -#include <string> +#include <stdint.h> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; @@ -17,8 +14,6 @@ typedef int64_t CAmount; static const CAmount COIN = 100000000; static const CAmount CENT = 1000000; -extern const std::string CURRENCY_UNIT; - /** No amount larger than this (in satoshi) is valid. * * Note that this constant is *not* the total money supply, which in Bitcoin @@ -31,42 +26,4 @@ extern const std::string CURRENCY_UNIT; static const CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -/** - * Fee rate in satoshis per kilobyte: CAmount / kB - */ -class CFeeRate -{ -private: - CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes -public: - /** Fee rate of 0 satoshis per kB */ - CFeeRate() : nSatoshisPerK(0) { } - explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } - /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ - CFeeRate(const CAmount& nFeePaid, size_t nBytes); - CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } - /** - * Return the fee in satoshis for the given size in bytes. - */ - CAmount GetFee(size_t nBytes) const; - /** - * Return the fee in satoshis for a size of 1000 bytes - */ - CAmount GetFeePerK() const { return GetFee(1000); } - friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } - friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } - friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } - friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } - friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } - CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } - std::string ToString() const; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } -}; - #endif // BITCOIN_AMOUNT_H diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index b0df3d2b04..33631d2d15 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,6 +5,7 @@ #include "bench.h" #include "perf.h" +#include <assert.h> #include <iostream> #include <iomanip> #include <sys/time.h> diff --git a/src/bench/bench.h b/src/bench/bench.h index f12a41126c..1f36f2a4bc 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <functional> +#include <limits> #include <map> #include <string> -#include <boost/function.hpp> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> @@ -59,7 +60,7 @@ namespace benchmark { bool KeepRunning(); }; - typedef boost::function<void(State&)> BenchFunction; + typedef std::function<void(State&)> BenchFunction; class BenchRunner { diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1e8e3d462f..5aab3381fd 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50 * CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21 * CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22 * CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index c6c932454a..195388839e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -40,7 +40,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) char a = '\0'; stream.write(&a, 1); // Prevent compaction - Consensus::Params params = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); while (state.KeepRunning()) { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here @@ -48,7 +48,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) assert(stream.Rewind(sizeof(block_bench::block413567))); CValidationState validationState; - assert(CheckBlock(block, validationState, params)); + assert(CheckBlock(block, validationState, chainParams->GetConsensus())); } } diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 5edd43d41e..885b787b4d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -30,6 +30,8 @@ static const int CONTINUE_EXECUTION=-1; std::string HelpMessageCli() { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); std::string strUsage; strUsage += HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); @@ -38,7 +40,7 @@ std::string HelpMessageCli() AppendParamsHelpMessages(strUsage); strUsage += HelpMessageOpt("-named", strprintf(_("Pass named instead of positional arguments (default: %s)"), DEFAULT_NAMED)); strUsage += HelpMessageOpt("-rpcconnect=<ip>", strprintf(_("Send commands to node running on <ip> (default: %s)"), DEFAULT_RPCCONNECT)); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 45738b5df8..cf280f485c 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -556,24 +556,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw std::runtime_error(err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]); + newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and private keys given, @@ -595,13 +597,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (!coins || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { fComplete = false; continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 31680a8ec7..5922e45801 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -117,17 +117,14 @@ bool AppInit(int argc, char* argv[]) return false; } - // Command-line RPC - bool fCommandLine = false; - for (int i = 1; i < argc; i++) - if (!IsSwitchChar(argv[i][0]) && !boost::algorithm::istarts_with(argv[i], "bitcoin:")) - fCommandLine = true; - - if (fCommandLine) - { - fprintf(stderr, "Error: There is no RPC client functionality in bitcoind anymore. Use the bitcoin-cli utility instead.\n"); - exit(EXIT_FAILURE); + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + fprintf(stderr, "Error: Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]); + exit(EXIT_FAILURE); + } } + // -server defaults to true for bitcoind but not for the GUI so do this here SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console diff --git a/src/bloom.cpp b/src/bloom.cpp index ac3e565721..7ed982c984 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -19,7 +19,7 @@ #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** * The ideal size for a bloom filter with a given number of elements and false positive rate is: * - nElements * log(fp rate) / ln(2)^2 @@ -40,7 +40,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int } // Private constructor used by CRollingBloomFilter -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn) : vData((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)) / 8), isFull(false), isEmpty(true), @@ -120,7 +120,7 @@ void CBloomFilter::clear() isEmpty = true; } -void CBloomFilter::reset(unsigned int nNewTweak) +void CBloomFilter::reset(const unsigned int nNewTweak) { clear(); nTweak = nNewTweak; @@ -214,7 +214,7 @@ void CBloomFilter::UpdateEmptyFull() isEmpty = empty; } -CRollingBloomFilter::CRollingBloomFilter(unsigned int nElements, double fpRate) +CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate) { double logFpRate = log(fpRate); /* The optimal number of hash functions is log(fpRate) / log(0.5), but diff --git a/src/bloom.h b/src/bloom.h index 5ad727c330..7ca9682239 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -54,7 +54,7 @@ private: unsigned int Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const; // Private constructor for CRollingBloomFilter, no restrictions on size - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak); friend class CRollingBloomFilter; public: @@ -67,7 +67,7 @@ public: * It should generally always be a random value (and is largely only exposed for unit testing) * nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) */ - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} ADD_SERIALIZE_METHODS; @@ -89,7 +89,7 @@ public: bool contains(const uint256& hash) const; void clear(); - void reset(unsigned int nNewTweak); + void reset(const unsigned int nNewTweak); //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) @@ -122,7 +122,7 @@ public: // A random bloom filter calls GetRand() at creation time. // Don't create global CRollingBloomFilter objects, as they may be // constructed before the randomizer is properly initialized. - CRollingBloomFilter(unsigned int nElements, double nFPRate); + CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); void insert(const uint256& hash); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 1dc29826af..5055fb3e0a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -55,6 +55,12 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits return CreateGenesisBlock(pszTimestamp, genesisOutputScript, nTime, nNonce, nBits, nVersion, genesisReward); } +void CChainParams::UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +{ + consensus.vDeployments[d].nStartTime = nStartTime; + consensus.vDeployments[d].nTimeout = nTimeout; +} + /** * Main network */ @@ -165,7 +171,6 @@ public: }; } }; -static CMainParams mainParams; /** * Testnet (v3) @@ -253,7 +258,6 @@ public: } }; -static CTestNetParams testNetParams; /** * Regression test @@ -326,42 +330,34 @@ public: base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >(); base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >(); } - - void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) - { - consensus.vDeployments[d].nStartTime = nStartTime; - consensus.vDeployments[d].nTimeout = nTimeout; - } }; -static CRegTestParams regTestParams; -static CChainParams *pCurrentParams = 0; +static std::unique_ptr<CChainParams> globalChainParams; const CChainParams &Params() { - assert(pCurrentParams); - return *pCurrentParams; + assert(globalChainParams); + return *globalChainParams; } -CChainParams& Params(const std::string& chain) +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CChainParams>(new CMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CChainParams>(new CTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; - else - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + return std::unique_ptr<CChainParams>(new CRegTestParams()); + throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectParams(const std::string& network) { SelectBaseParams(network); - pCurrentParams = &Params(network); + globalChainParams = CreateChainParams(network); } -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) { - regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout); + globalChainParams->UpdateBIP9Parameters(d, nStartTime, nTimeout); } diff --git a/src/chainparams.h b/src/chainparams.h index 4fe88c691c..e5312d1080 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -11,6 +11,7 @@ #include "primitives/block.h" #include "protocol.h" +#include <memory> #include <vector> struct CDNSSeedData { @@ -75,6 +76,7 @@ public: const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } const ChainTxData& TxData() const { return chainTxData; } + void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); protected: CChainParams() {} @@ -95,15 +97,17 @@ protected: }; /** - * Return the currently selected parameters. This won't change after app - * startup, except for unit tests. + * Creates and returns a std::unique_ptr<CChainParams> of the chosen chain. + * @returns a CChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. */ -const CChainParams &Params(); +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain); /** - * @returns CChainParams for the given BIP70 chain name. + * Return the currently selected parameters. This won't change after app + * startup, except for unit tests. */ -CChainParams& Params(const std::string& chain); +const CChainParams &Params(); /** * Sets the params returned by Params() to those for the given BIP70 chain name. @@ -114,6 +118,6 @@ void SelectParams(const std::string& chain); /** * Allows modifying the BIP9 regtest parameters. */ -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); +void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index d013cc1450..43c9a13c54 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -35,7 +35,6 @@ public: nRPCPort = 8332; } }; -static CBaseMainParams mainParams; /** * Testnet (v3) @@ -49,7 +48,6 @@ public: strDataDir = "testnet3"; } }; -static CBaseTestNetParams testNetParams; /* * Regression test @@ -63,31 +61,30 @@ public: strDataDir = "regtest"; } }; -static CBaseRegTestParams regTestParams; -static CBaseChainParams* pCurrentBaseParams = 0; +static std::unique_ptr<CBaseChainParams> globalChainBaseParams; const CBaseChainParams& BaseParams() { - assert(pCurrentBaseParams); - return *pCurrentBaseParams; + assert(globalChainBaseParams); + return *globalChainBaseParams; } -CBaseChainParams& BaseParams(const std::string& chain) +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CBaseChainParams>(new CBaseMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CBaseChainParams>(new CBaseTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; + return std::unique_ptr<CBaseChainParams>(new CBaseRegTestParams()); else throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectBaseParams(const std::string& chain) { - pCurrentBaseParams = &BaseParams(chain); + globalChainBaseParams = CreateBaseChainParams(chain); } std::string ChainNameFromCommandLine() diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 84350cf65b..fc101f5b77 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CHAINPARAMSBASE_H #define BITCOIN_CHAINPARAMSBASE_H +#include <memory> #include <string> #include <vector> @@ -31,6 +32,13 @@ protected: }; /** + * Creates and returns a std::unique_ptr<CBaseChainParams> of the chosen chain. + * @returns a CBaseChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. + */ +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain); + +/** * Append the help messages for the chainparams options to the * parameter string. */ @@ -42,8 +50,6 @@ void AppendParamsHelpMessages(std::string& strUsage, bool debugHelp=true); */ const CBaseChainParams& BaseParams(); -CBaseChainParams& BaseParams(const std::string& chain); - /** Sets the params returned by Params() to those for the given network. */ void SelectBaseParams(const std::string& chain); diff --git a/src/coins.cpp b/src/coins.cpp index b2e33abf33..5b7c562678 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,174 +4,126 @@ #include "coins.h" +#include "consensus/consensus.h" #include "memusage.h" #include "random.h" #include <assert.h> -/** - * calculate number of bytes for the bitmask, and its number of non-zero bytes - * each bit in the bitmask represents the availability of one output, but the - * availabilities of the first two outputs are encoded separately - */ -void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const { - unsigned int nLastUsedByte = 0; - for (unsigned int b = 0; 2+b*8 < vout.size(); b++) { - bool fZero = true; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) { - if (!vout[2+b*8+i].IsNull()) { - fZero = false; - continue; - } - } - if (!fZero) { - nLastUsedByte = b + 1; - nNonzeroBytes++; - } - } - nBytes += nLastUsedByte; -} - -bool CCoins::Spend(uint32_t nPos) -{ - if (nPos >= vout.size() || vout[nPos].IsNull()) - return false; - vout[nPos].SetNull(); - Cleanup(); - return true; -} - -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } +size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } - -CCoinsViewCache::~CCoinsViewCache() -{ - assert(!hasModifier); -} +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; - CCoins tmp; - if (!base->GetCoins(txid, tmp)) + Coin tmp; + if (!base->GetCoin(outpoint, tmp)) return cacheCoins.end(); - CCoinsMap::iterator ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); - if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our + CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; + if (ret->second.coin.IsSpent()) { + // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it != cacheCoins.end()) { - coins = it->second.coins; + coin = it->second.coin; return true; } return false; } -CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - size_t cachedCoinUsage = 0; - if (ret.second) { - if (!base->GetCoins(txid, ret.first->second.coins)) { - // The parent view does not have this entry; mark it as fresh. - ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } else if (ret.first->second.coins.IsPruned()) { - // The parent view only has a pruned entry for this; mark it as fresh. - ret.first->second.flags = CCoinsCacheEntry::FRESH; +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { + assert(!coin.IsSpent()); + if (coin.out.scriptPubKey.IsUnspendable()) return; + CCoinsMap::iterator it; + bool inserted; + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); + bool fresh = false; + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } + if (!possible_overwrite) { + if (!it->second.coin.IsSpent()) { + throw std::logic_error("Adding new coin that replaces non-pruned entry"); } - } else { - cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); + } + it->second.coin = std::move(coin); + it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); +} + +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { + bool fCoinbase = tx.IsCoinBase(); + const uint256& txid = tx.GetHash(); + for (size_t i = 0; i < tx.vout.size(); ++i) { + // Pass fCoinbase as the possible_overwrite flag to AddCoin, in order to correctly + // deal with the pre-BIP30 occurrances of duplicate coinbase transactions. + cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), fCoinbase); } - // Assume that whenever ModifyCoins is called, the entry will be modified. - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, cachedCoinUsage); } -/* ModifyNewCoins allows for faster coin modification when creating the new - * outputs from a transaction. It assumes that BIP 30 (no duplicate txids) - * applies and has already been tested for (or the test is not required due to - * BIP 34, height in coinbase). If we can assume BIP 30 then we know that any - * non-coinbase transaction we are adding to the UTXO must not already exist in - * the utxo unless it is fully spent. Thus we can check only if it exists DIRTY - * at the current level of the cache, in which case it is not safe to mark it - * FRESH (b/c then its spentness still needs to flushed). If it's not dirty and - * doesn't exist or is pruned in the current cache, we know it either doesn't - * exist or is pruned in parent caches, which is the definition of FRESH. The - * exception to this is the two historical violations of BIP 30 in the chain, - * both of which were coinbases. We do not mark these fresh so we we can ensure - * that they will still be properly overwritten when spent. - */ -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - if (!coinbase) { - // New coins must not already exist. - if (!ret.first->second.coins.IsPruned()) - throw std::logic_error("ModifyNewCoins should not find pre-existing coins on a non-coinbase unless they are pruned!"); - - if (!(ret.first->second.flags & CCoinsCacheEntry::DIRTY)) { - // If the coin is known to be pruned (have no unspent outputs) in - // the current view and the cache entry is not dirty, we know the - // coin also must be pruned in the parent view as well, so it is safe - // to mark this fresh. - ret.first->second.flags |= CCoinsCacheEntry::FRESH; - } +void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) return; + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + if (moveout) { + *moveout = std::move(it->second.coin); + } + if (it->second.flags & CCoinsCacheEntry::FRESH) { + cacheCoins.erase(it); + } else { + it->second.flags |= CCoinsCacheEntry::DIRTY; + it->second.coin.Clear(); } - ret.first->second.coins.Clear(); - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, 0); } -const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +static const Coin coinEmpty; + +const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { - return NULL; + return coinEmpty; } else { - return &it->second.coins; + return it->second.coin; } } -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } -bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { - CCoinsMap::const_iterator it = cacheCoins.find(txid); +bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -186,19 +138,18 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coins.IsPruned())) { + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache @@ -211,21 +162,21 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coins.IsPruned()) + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -249,11 +200,11 @@ bool CCoinsViewCache::Flush() { return fOk; } -void CCoinsViewCache::Uncache(const uint256& hash) +void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -262,13 +213,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const -{ - const CCoins* coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; -} - CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const { if (tx.IsCoinBase()) @@ -276,7 +220,7 @@ CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const CAmount nResult = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += GetOutputFor(tx.vin[i]).nValue; + nResult += AccessCoin(tx.vin[i].prevout).out.nValue; return nResult; } @@ -285,9 +229,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -295,25 +237,15 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { - assert(!cache.hasModifier); - cache.hasModifier = true; -} +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. -CCoinsModifier::~CCoinsModifier() +const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { - assert(cache.hasModifier); - cache.hasModifier = false; - it->second.coins.Cleanup(); - cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage - if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { - cache.cacheCoins.erase(it); - } else { - // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + COutPoint iter(txid, 0); + while (iter.n < MAX_OUTPUTS_PER_BLOCK) { + const Coin& alternate = view.AccessCoin(iter); + if (!alternate.IsSpent()) return alternate; + ++iter.n; } -} - -CCoinsViewCursor::~CCoinsViewCursor() -{ + return coinEmpty; } diff --git a/src/coins.h b/src/coins.h index 065bae56e9..476db8f37c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -20,135 +20,37 @@ #include <boost/foreach.hpp> #include <unordered_map> -/** - * Pruned version of CTransaction: only retains metadata and unspent transaction outputs +/** + * A UTXO entry. * * Serialized format: - * - VARINT(nVersion) - * - VARINT(nCode) - * - unspentness bitvector, for vout[2] and further; least significant byte first - * - the non-spent CTxOuts (via CTxOutCompressor) - * - VARINT(nHeight) - * - * The nCode value consists of: - * - bit 0: IsCoinBase() - * - bit 1: vout[0] is not spent - * - bit 2: vout[1] is not spent - * - The higher bits encode N, the number of non-zero bytes in the following bitvector. - * - In case both bit 1 and bit 2 are unset, they encode N-1, as there must be at - * least one non-spent output). - * - * Example: 0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e - * <><><--------------------------------------------><----> - * | \ | / - * version code vout[1] height - * - * - version = 1 - * - code = 4 (vout[1] is not spent, and 0 non-zero bytes of bitvector follow) - * - unspentness bitvector: as 0 non-zero bytes follow, it has length 0 - * - vout[1]: 835800816115944e077fe7c803cfa57f29b36bf87c1d35 - * * 8358: compact amount representation for 60000000000 (600 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 816115944e077fe7c803cfa57f29b36bf87c1d35: address uint160 - * - height = 203998 - * - * - * Example: 0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b - * <><><--><--------------------------------------------------><----------------------------------------------><----> - * / \ \ | | / - * version code unspentness vout[4] vout[16] height - * - * - version = 1 - * - code = 9 (coinbase, neither vout[0] or vout[1] are unspent, - * 2 (1, +1 because both bit 1 and bit 2 are unset) non-zero bitvector bytes follow) - * - unspentness bitvector: bits 2 (0x04) and 14 (0x4000) are set, so vout[2+2] and vout[14+2] are unspent - * - vout[4]: 86ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4ee - * * 86ef97d579: compact amount representation for 234925952 (2.35 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 61b01caab50f1b8e9c50a5057eb43c2d9563a4ee: address uint160 - * - vout[16]: bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4 - * * bbd123: compact amount representation for 110397 (0.001 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 8c988f1a4a4de2161e0f50aac7f17e7f9555caa4: address uint160 - * - height = 120891 + * - VARINT((coinbase ? 1 : 0) | (height << 1)) + * - the non-spent CTxOut (via CTxOutCompressor) */ -class CCoins +class Coin { public: - //! whether transaction is a coinbase - bool fCoinBase; - - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector<CTxOut> vout; + //! unspent transaction output + CTxOut out; - //! at which height this transaction was included in the active block chain - int nHeight; + //! whether containing transaction was a coinbase + unsigned int fCoinBase : 1; - //! version of the CTransaction; accesses to this value should probably check for nHeight as well, - //! as new tx version will probably only be introduced at certain heights - int nVersion; + //! at which height this containing transaction was included in the active block chain + uint32_t nHeight : 31; - void FromTx(const CTransaction &tx, int nHeightIn) { - fCoinBase = tx.IsCoinBase(); - vout = tx.vout; - nHeight = nHeightIn; - nVersion = tx.nVersion; - ClearUnspendable(); - } - - //! construct a CCoins from a CTransaction, at a given height - CCoins(const CTransaction &tx, int nHeightIn) { - FromTx(tx, nHeightIn); - } + //! construct a Coin from a CTxOut and height/coinbase information. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn),nHeight(nHeightIn) {} void Clear() { + out.SetNull(); fCoinBase = false; - std::vector<CTxOut>().swap(vout); nHeight = 0; - nVersion = 0; } //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0), nVersion(0) { } - - //!remove spent outputs at the end of vout - void Cleanup() { - while (vout.size() > 0 && vout.back().IsNull()) - vout.pop_back(); - if (vout.empty()) - std::vector<CTxOut>().swap(vout); - } - - void ClearUnspendable() { - BOOST_FOREACH(CTxOut &txout, vout) { - if (txout.scriptPubKey.IsUnspendable()) - txout.SetNull(); - } - Cleanup(); - } - - void swap(CCoins &to) { - std::swap(to.fCoinBase, fCoinBase); - to.vout.swap(vout); - std::swap(to.nHeight, nHeight); - std::swap(to.nVersion, nVersion); - } - - //! equality test - friend bool operator==(const CCoins &a, const CCoins &b) { - // Empty CCoins objects are always equal. - if (a.IsPruned() && b.IsPruned()) - return true; - return a.fCoinBase == b.fCoinBase && - a.nHeight == b.nHeight && - a.nVersion == b.nVersion && - a.vout == b.vout; - } - friend bool operator!=(const CCoins &a, const CCoins &b) { - return !(a == b); - } - - void CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const; + Coin() : fCoinBase(false), nHeight(0) { } bool IsCoinBase() const { return fCoinBase; @@ -156,115 +58,52 @@ public: template<typename Stream> void Serialize(Stream &s) const { - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - ::Serialize(s, VARINT(this->nVersion)); - // header code - ::Serialize(s, VARINT(nCode)); - // spentness bitmask - for (unsigned int b = 0; b<nMaskSize; b++) { - unsigned char chAvail = 0; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) - if (!vout[2+b*8+i].IsNull()) - chAvail |= (1 << i); - ::Serialize(s, chAvail); - } - // txouts themself - for (unsigned int i = 0; i < vout.size(); i++) { - if (!vout[i].IsNull()) - ::Serialize(s, CTxOutCompressor(REF(vout[i]))); - } - // coinbase height - ::Serialize(s, VARINT(nHeight)); + assert(!IsSpent()); + uint32_t code = nHeight * 2 + fCoinBase; + ::Serialize(s, VARINT(code)); + ::Serialize(s, CTxOutCompressor(REF(out))); } template<typename Stream> void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - ::Unserialize(s, VARINT(this->nVersion)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector<bool> vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); - } - // coinbase height - ::Unserialize(s, VARINT(nHeight)); - Cleanup(); + uint32_t code = 0; + ::Unserialize(s, VARINT(code)); + nHeight = code >> 1; + fCoinBase = code & 1; + ::Unserialize(s, REF(CTxOutCompressor(out))); } - //! mark a vout spent - bool Spend(uint32_t nPos); - - //! check whether a particular output is still available - bool IsAvailable(unsigned int nPos) const { - return (nPos < vout.size() && !vout[nPos].IsNull()); - } - - //! check whether the entire CCoins is spent - //! note that only !IsPruned() CCoins can be serialized - bool IsPruned() const { - BOOST_FOREACH(const CTxOut &out, vout) - if (!out.IsNull()) - return false; - return true; + bool IsSpent() const { + return out.IsNull(); } size_t DynamicMemoryUsage() const { - size_t ret = memusage::DynamicUsage(vout); - BOOST_FOREACH(const CTxOut &out, vout) { - ret += RecursiveDynamicUsage(out.scriptPubKey); - } - return ret; + return memusage::DynamicUsage(out.scriptPubKey); } }; -class SaltedTxidHasher +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint& id) const { + return SipHashUint256Extra(k0, k1, id.hash, id.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + Coin coin; // The actual cached data. unsigned char flags; enum Flags { @@ -277,20 +116,21 @@ struct CCoinsCacheEntry */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} }; -typedef std::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; +typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -306,17 +146,17 @@ private: class CCoinsView { public: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; + //! Retrieve the Coin (unspent transaction output) for a given outpoint. + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; + //! Just check whether we have data for a given outpoint. + //! This may (but cannot always) return true for spent outputs. + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -325,6 +165,9 @@ public: //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} + + //! Estimate database size (0 if not implemented) + virtual size_t EstimateSize() const { return 0; } }; @@ -336,45 +179,20 @@ protected: public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + size_t EstimateSize() const override; }; -class CCoinsViewCache; - -/** - * A reference to a mutable cache entry. Encapsulating it allows us to run - * cleanup code after the modification is finished, and keeping track of - * concurrent modifications. - */ -class CCoinsModifier -{ -private: - CCoinsViewCache& cache; - CCoinsMap::iterator it; - size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); - -public: - CCoins* operator->() { return &it->second.coins; } - CCoins& operator*() { return it->second.coins; } - ~CCoinsModifier(); - friend class CCoinsViewCache; -}; - /** CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: - /* Whether this cache has an active modifier. */ - bool hasModifier; - - /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -382,51 +200,45 @@ protected: mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); - ~CCoinsViewCache(); // Standard CCoinsView methods - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); /** - * Check if we have the given tx already loaded in this cache. - * The semantics are the same as HaveCoins(), but no calls to + * Check if we have the given utxo already loaded in this cache. + * The semantics are the same as HaveCoin(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const uint256 &txid) const; + bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a pointer to CCoins in the cache, or NULL if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are + * Return a reference to Coin in the cache, or a pruned one if not found. This is + * more efficient than GetCoin. Modifications to other cache entries are * allowed while accessing the returned pointer. */ - const CCoins* AccessCoins(const uint256 &txid) const; + const Coin& AccessCoin(const COutPoint &output) const; /** - * Return a modifiable reference to a CCoins. If no entry with the given - * txid exists, a new one is created. Simultaneous modifications are not - * allowed. + * Add a coin. Set potential_overwrite to true if a non-pruned version may + * already exist. */ - CCoinsModifier ModifyCoins(const uint256 &txid); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); /** - * Return a modifiable reference to a CCoins. Assumes that no entry with the given - * txid exists and creates a new one. This saves a database access in the case where - * the coins were to be wiped out by FromTx anyway. This should not be called with - * the 2 historical coinbase duplicate pairs because the new coins are marked fresh, and - * in the event the duplicate coinbase was spent before a flush, the now pruned coins - * would not properly overwrite the first coinbase of the pair. Simultaneous modifications - * are not allowed. + * Spend a coin. Pass moveto in order to get the deleted data. + * If no unspent output exists for the passed outpoint, this call + * has no effect. */ - CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); + void SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** * Push the modifications applied to this cache to its base. @@ -436,12 +248,12 @@ public: bool Flush(); /** - * Removes the transaction with the given hash from the cache, if it is + * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ - void Uncache(const uint256 &txid); + void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -460,12 +272,8 @@ public: //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - const CTxOut &GetOutputFor(const CTxIn& input) const; - - friend class CCoinsModifier; - private: - CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -473,4 +281,13 @@ private: CCoinsViewCache(const CCoinsViewCache &); }; +//! Utility function to add all of a transaction's outputs to a cache. +// It assumes that overwrites are only possible for coinbase transactions, +// TODO: pass in a boolean to limit these possible overwrites to known +// (pre-BIP34) cases. +void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); + +//! Utility function to find any unspent output with a given txid. +const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); + #endif // BITCOIN_COINS_H diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp new file mode 100644 index 0000000000..bf68f8754b --- /dev/null +++ b/src/consensus/tx_verify.cpp @@ -0,0 +1,246 @@ +// Copyright (c) 2017-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "tx_verify.h" + +#include "consensus.h" +#include "primitives/transaction.h" +#include "script/interpreter.h" +#include "validation.h" + +// TODO remove the following dependencies +#include "chain.h" +#include "coins.h" +#include "utilmoneystr.h" + +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) +{ + if (tx.nLockTime == 0) + return true; + if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) + return true; + for (const auto& txin : tx.vin) { + if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) + return false; + } + return true; +} + +std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + assert(prevHeights->size() == tx.vin.size()); + + // Will be set to the equivalent height- and time-based nLockTime + // values that would be necessary to satisfy all relative lock- + // time constraints given our view of block chain history. + // The semantics of nLockTime are the last invalid height/time, so + // use -1 to have the effect of any height or time being valid. + int nMinHeight = -1; + int64_t nMinTime = -1; + + // tx.nVersion is signed integer so requires cast to unsigned otherwise + // we would be doing a signed comparison and half the range of nVersion + // wouldn't support BIP 68. + bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 + && flags & LOCKTIME_VERIFY_SEQUENCE; + + // Do not enforce sequence numbers as a relative lock time + // unless we have been instructed to + if (!fEnforceBIP68) { + return std::make_pair(nMinHeight, nMinTime); + } + + for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { + const CTxIn& txin = tx.vin[txinIndex]; + + // Sequence numbers with the most significant bit set are not + // treated as relative lock-times, nor are they given any + // consensus-enforced meaning at this point. + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { + // The height of this input is not relevant for sequence locks + (*prevHeights)[txinIndex] = 0; + continue; + } + + int nCoinHeight = (*prevHeights)[txinIndex]; + + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { + int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); + // NOTE: Subtract 1 to maintain nLockTime semantics + // BIP 68 relative lock times have the semantics of calculating + // the first block or time at which the transaction would be + // valid. When calculating the effective block time or height + // for the entire transaction, we switch to using the + // semantics of nLockTime which is the last invalid block + // time or height. Thus we subtract 1 from the calculated + // time or height. + + // Time-based relative lock-times are measured from the + // smallest allowed timestamp of the block containing the + // txout being spent, which is the median time past of the + // block prior. + nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); + } else { + nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); + } + } + + return std::make_pair(nMinHeight, nMinTime); +} + +bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) +{ + assert(block.pprev); + int64_t nBlockTime = block.pprev->GetMedianTimePast(); + if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) + return false; + + return true; +} + +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); +} + +unsigned int GetLegacySigOpCount(const CTransaction& tx) +{ + unsigned int nSigOps = 0; + for (const auto& txin : tx.vin) + { + nSigOps += txin.scriptSig.GetSigOpCount(false); + } + for (const auto& txout : tx.vout) + { + nSigOps += txout.scriptPubKey.GetSigOpCount(false); + } + return nSigOps; +} + +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) +{ + if (tx.IsCoinBase()) + return 0; + + unsigned int nSigOps = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; + if (prevout.scriptPubKey.IsPayToScriptHash()) + nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); + } + return nSigOps; +} + +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +{ + int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; + + if (tx.IsCoinBase()) + return nSigOps; + + if (flags & SCRIPT_VERIFY_P2SH) { + nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; + } + + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); + } + return nSigOps; +} + +bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) +{ + // Basic checks that don't depend on any context + if (tx.vin.empty()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); + if (tx.vout.empty()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); + // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) + if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); + + // Check for negative or overflow output values + CAmount nValueOut = 0; + for (const auto& txout : tx.vout) + { + if (txout.nValue < 0) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); + if (txout.nValue > MAX_MONEY) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); + nValueOut += txout.nValue; + if (!MoneyRange(nValueOut)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); + } + + // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock + if (fCheckDuplicateInputs) { + std::set<COutPoint> vInOutPoints; + for (const auto& txin : tx.vin) + { + if (!vInOutPoints.insert(txin.prevout).second) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); + } + } + + if (tx.IsCoinBase()) + { + if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) + return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); + } + else + { + for (const auto& txin : tx.vin) + if (txin.prevout.IsNull()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); + } + + return true; +} + +bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) +{ + // This doesn't trigger the DoS code on purpose; if it did, it would make it easier + // for an attacker to attempt to split the network. + if (!inputs.HaveInputs(tx)) + return state.Invalid(false, 0, "", "Inputs unavailable"); + + CAmount nValueIn = 0; + CAmount nFees = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const COutPoint &prevout = tx.vin[i].prevout; + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // If prev is coinbase, check that it's matured + if (coin.IsCoinBase()) { + if (nSpendHeight - coin.nHeight < COINBASE_MATURITY) + return state.Invalid(false, + REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); + } + + // Check for negative or overflow input values + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + + } + + if (nValueIn < tx.GetValueOut()) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, + strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); + + // Tally transaction fees + CAmount nTxFee = nValueIn - tx.GetValueOut(); + if (nTxFee < 0) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); + nFees += nTxFee; + if (!MoneyRange(nFees)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + return true; +} diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h new file mode 100644 index 0000000000..d46d3294ca --- /dev/null +++ b/src/consensus/tx_verify.h @@ -0,0 +1,78 @@ +// Copyright (c) 2017-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CONSENSUS_TX_VERIFY_H +#define BITCOIN_CONSENSUS_TX_VERIFY_H + +#include <stdint.h> +#include <vector> + +class CBlockIndex; +class CCoinsViewCache; +class CTransaction; +class CValidationState; + +/** Transaction validation functions */ + +/** Context-independent validity checks */ +bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); + +namespace Consensus { +/** + * Check whether all inputs of this transaction are valid (no double spends and amounts) + * This does not modify the UTXO set. This does not check scripts and sigs. + * Preconditions: tx.IsCoinBase() is false. + */ +bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); +} // namespace Consensus + +/** Auxiliary functions for transaction validation (ideally should not be exposed) */ + +/** + * Count ECDSA signature operations the old-fashioned (pre-0.6) way + * @return number of sigops this transaction's outputs will produce when spent + * @see CTransaction::FetchInputs + */ +unsigned int GetLegacySigOpCount(const CTransaction& tx); + +/** + * Count ECDSA signature operations in pay-to-script-hash inputs. + * + * @param[in] mapInputs Map of previous transactions that have outputs we're spending + * @return maximum number of sigops required to validate this transaction's inputs + * @see CTransaction::FetchInputs + */ +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& mapInputs); + +/** + * Compute total signature operation cost of a transaction. + * @param[in] tx Transaction for which we are computing the cost + * @param[in] inputs Map of previous transactions that have outputs we're spending + * @param[out] flags Script verification flags + * @return Total signature operation cost of tx + */ +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); + +/** + * Check if transaction is final and can be included in a block with the + * specified height and time. Consensus critical. + */ +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); + +/** + * Calculates the block height and previous block's median time past at + * which the transaction will be considered final in the context of BIP 68. + * Also removes from the vector of input heights any entries which did not + * correspond to sequence locked inputs as they do not affect the calculation. + */ +std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); + +bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair); +/** + * Check if transaction is final per BIP 68 sequence numbers and can be included in a block. + * Consensus critical. Takes as input a list of heights at which tx's inputs (in order) confirmed. + */ +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); + +#endif // BITCOIN_CONSENSUS_TX_VERIFY_H diff --git a/src/core_memusage.h b/src/core_memusage.h index 5e10182075..e4ccd54c42 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -63,4 +63,9 @@ static inline size_t RecursiveDynamicUsage(const CBlockLocator& locator) { return memusage::DynamicUsage(locator.vHave); } +template<typename X> +static inline size_t RecursiveDynamicUsage(const std::shared_ptr<X>& p) { + return p ? memusage::DynamicUsage(p) + RecursiveDynamicUsage(*p) : 0; +} + #endif // BITCOIN_CORE_MEMUSAGE_H diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b13e98b7a4..24ef71bfbf 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -55,11 +55,19 @@ private: CDataStream ssKey; CDataStream ssValue; + size_t size_estimate; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + + void Clear() + { + batch.Clear(); + size_estimate = 0; + } template <typename K, typename V> void Write(const K& key, const V& value) @@ -74,6 +82,14 @@ public: leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } @@ -86,8 +102,16 @@ public: leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } + + size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator @@ -281,7 +305,22 @@ public: * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); + + template<typename K> + size_t EstimateSize(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; + } }; #endif // BITCOIN_DBWRAPPER_H - @@ -21,4 +21,4 @@ namespace fsbridge { FILE *freopen(const fs::path& p, const char *mode, FILE *stream); }; -#endif +#endif // BITCOIN_FS_H diff --git a/src/hash.cpp b/src/hash.cpp index a14a2386a2..b361c90d16 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -208,3 +208,44 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } + +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = (((uint64_t)36) << 56) | extra; + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index eacb8f04fe..b9952d39fc 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,6 +160,41 @@ public: } }; +/** Reads data from an underlying stream, while hashing the read data. */ +template<typename Source> +class CHashVerifier : public CHashWriter +{ +private: + Source* source; + +public: + CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} + + void read(char* pch, size_t nSize) + { + source->read(pch, nSize); + this->write(pch, nSize); + } + + void ignore(size_t nSize) + { + char data[1024]; + while (nSize > 0) { + size_t now = std::min<size_t>(nSize, 1024); + read(data, now); + nSize -= now; + } + } + + template<typename T> + CHashVerifier<Source>& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template<typename T> uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) @@ -206,5 +241,6 @@ public: * .Finalize() */ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra); #endif // BITCOIN_HASH_H diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 21c64c5c83..5ab6d8d732 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -30,7 +30,7 @@ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; class HTTPRPCTimer : public RPCTimerBase { public: - HTTPRPCTimer(struct event_base* eventBase, boost::function<void(void)>& func, int64_t millis) : + HTTPRPCTimer(struct event_base* eventBase, std::function<void(void)>& func, int64_t millis) : ev(eventBase, false, func) { struct timeval tv; @@ -52,7 +52,7 @@ public: { return "HTTP"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new HTTPRPCTimer(base, func, millis); } @@ -93,9 +93,9 @@ static bool multiUserAuthorized(std::string strUserPass) std::string strUser = strUserPass.substr(0, strUserPass.find(":")); std::string strPass = strUserPass.substr(strUserPass.find(":") + 1); - if (mapMultiArgs.count("-rpcauth") > 0) { + if (gArgs.IsArgSet("-rpcauth")) { //Search for multi-user login/pass "rpcauth" from config - BOOST_FOREACH(std::string strRPCAuth, mapMultiArgs.at("-rpcauth")) + BOOST_FOREACH(std::string strRPCAuth, gArgs.GetArgs("-rpcauth")) { std::vector<std::string> vFields; boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index e7df23295d..0d1cba3fd2 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -196,9 +196,8 @@ static bool InitHTTPAllowList() LookupHost("::1", localv6, false); rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost - if (mapMultiArgs.count("-rpcallowip")) { - const std::vector<std::string>& vAllow = mapMultiArgs.at("-rpcallowip"); - for (std::string strAllow : vAllow) { + if (gArgs.IsArgSet("-rpcallowip")) { + for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { @@ -321,12 +320,11 @@ static bool HTTPBindAddresses(struct evhttp* http) if (IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); } - } else if (mapMultiArgs.count("-rpcbind")) { // Specific bind address - const std::vector<std::string>& vbind = mapMultiArgs.at("-rpcbind"); - for (std::vector<std::string>::const_iterator i = vbind.begin(); i != vbind.end(); ++i) { + } else if (gArgs.IsArgSet("-rpcbind")) { // Specific bind address + for (const std::string& strRPCBind : gArgs.GetArgs("-rpcbind")) { int port = defaultPort; std::string host; - SplitHostPort(*i, port, host); + SplitHostPort(strRPCBind, port, host); endpoints.push_back(std::make_pair(host, port)); } } else { // No specific bind address specified, bind to any diff --git a/src/init.cpp b/src/init.cpp index 64f571f284..33023a1800 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -25,6 +25,7 @@ #include "netbase.h" #include "net.h" #include "net_processing.h" +#include "policy/feerate.h" #include "policy/fees.h" #include "policy/policy.h" #include "rpc/server.h" @@ -58,7 +59,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/bind.hpp> -#include <boost/function.hpp> #include <boost/interprocess/sync/file_lock.hpp> #include <boost/thread.hpp> #include <openssl/crypto.h> @@ -146,9 +146,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); @@ -208,11 +208,13 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); - if (fDumpMempoolLater) + if (fDumpMempoolLater && GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); + } if (fFeeEstimatesInitialized) { + ::feeEstimator.FlushUnconfirmed(::mempool); fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) @@ -327,6 +329,10 @@ void OnRPCPreCommand(const CRPCCommand& cmd) std::string HelpMessage(HelpMessageMode mode) { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); + const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); + const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); const bool showDebug = GetBoolArg("-help-debug", false); // When adding new options to the categories, please keep and ensure alphabetical ordering. @@ -338,7 +344,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-blocknotify=<cmd>", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); if (showDebug) strUsage += HelpMessageOpt("-blocksonly", strprintf(_("Whether to operate in a blocks only mode (default: %u)"), DEFAULT_BLOCKSONLY)); - strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), Params(CBaseChainParams::MAIN).GetConsensus().defaultAssumeValid.GetHex(), Params(CBaseChainParams::TESTNET).GetConsensus().defaultAssumeValid.GetHex())); + strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex())); strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); if (mode == HMM_BITCOIND) { @@ -354,6 +360,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt("-blockreconstructionextratxn=<n>", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); @@ -391,7 +398,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-onlynet=<net>", _("Only connect to nodes in network <net> (ipv4, ipv6 or onion)")); strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt("-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), DEFAULT_PEERBLOOMFILTERS)); - strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort())); + strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); @@ -428,8 +435,8 @@ std::string HelpMessage(HelpMessageMode mode) { strUsage += HelpMessageOpt("-checkblocks=<n>", strprintf(_("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); strUsage += HelpMessageOpt("-checklevel=<n>", strprintf(_("How thorough the block verification of -checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); - strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); - strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", defaultChainParams->DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", defaultChainParams->DefaultConsistencyChecks())); strUsage += HelpMessageOpt("-checkpoints", strprintf("Disable expensive verification for known chain history (default: %u)", DEFAULT_CHECKPOINTS_ENABLED)); strUsage += HelpMessageOpt("-disablesafemode", strprintf("Disable safemode, override a real safe mode event (default: %u)", DEFAULT_DISABLE_SAFEMODE)); strUsage += HelpMessageOpt("-testsafemode", strprintf("Force safe mode (default: %u)", DEFAULT_TESTSAFEMODE)); @@ -470,7 +477,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) { - strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !Params(CBaseChainParams::TESTNET).RequireStandard())); + strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", defaultChainParams->RequireStandard())); strUsage += HelpMessageOpt("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define cost of relay, used for mempool limiting and BIP 125 replacement. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE))); strUsage += HelpMessageOpt("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to defined dust, the value of an output such that it will cost about 1/3 of its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE))); } @@ -498,7 +505,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcauth=<userpw>", _("Username and hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcuser. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times")); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcallowip=<ip>", _("Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); @@ -679,8 +686,10 @@ void ThreadImport(std::vector<fs::path> vImportFiles) StartShutdown(); } } // End scope of CImportingNow - LoadMempool(); - fDumpMempoolLater = !fRequestShutdown; + if (GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + LoadMempool(); + fDumpMempoolLater = !fRequestShutdown; + } } /** Sanity checks @@ -737,7 +746,7 @@ void InitParameterInteraction() LogPrintf("%s: parameter interaction: -whitebind set -> setting -listen=1\n", __func__); } - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) { + if (gArgs.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen by default if (SoftSetBoolArg("-dnsseed", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -dnsseed=0\n", __func__); @@ -891,8 +900,8 @@ bool AppInitParameterInteraction() // Make sure enough file descriptors are available int nBind = std::max( - (mapMultiArgs.count("-bind") ? mapMultiArgs.at("-bind").size() : 0) + - (mapMultiArgs.count("-whitebind") ? mapMultiArgs.at("-whitebind").size() : 0), size_t(1)); + (gArgs.IsArgSet("-bind") ? gArgs.GetArgs("-bind").size() : 0) + + (gArgs.IsArgSet("-whitebind") ? gArgs.GetArgs("-whitebind").size() : 0), size_t(1)); nUserMaxConnections = GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); @@ -907,9 +916,9 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (mapMultiArgs.count("-debug") > 0) { + if (gArgs.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string>& categories = mapMultiArgs.at("-debug"); + const std::vector<std::string> categories = gArgs.GetArgs("-debug"); if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { for (const auto& cat : categories) { @@ -924,9 +933,8 @@ bool AppInitParameterInteraction() } // Now remove the logging categories which were explicitly excluded - if (mapMultiArgs.count("-debugexclude") > 0) { - const std::vector<std::string>& excludedCategories = mapMultiArgs.at("-debugexclude"); - for (const auto& cat : excludedCategories) { + if (gArgs.IsArgSet("-debugexclude")) { + for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { uint32_t flag = 0; if (!GetLogCategory(&flag, &cat)) { InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); @@ -1096,15 +1104,14 @@ bool AppInitParameterInteraction() fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } - if (mapMultiArgs.count("-bip9params")) { + if (gArgs.IsArgSet("-bip9params")) { // Allow overriding BIP9 parameters for testing if (!chainparams.MineBlocksOnDemand()) { return InitError("BIP9 parameters may only be overridden on regtest."); } - const std::vector<std::string>& deployments = mapMultiArgs.at("-bip9params"); - for (auto i : deployments) { + for (const std::string& strDeployment : gArgs.GetArgs("-bip9params")) { std::vector<std::string> vDeploymentParams; - boost::split(vDeploymentParams, i, boost::is_any_of(":")); + boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); if (vDeploymentParams.size() != 3) { return InitError("BIP9 parameters malformed, expecting deployment:start:end"); } @@ -1119,7 +1126,7 @@ bool AppInitParameterInteraction() for (int j=0; j<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[j].name) == 0) { - UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); + UpdateBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); found = true; LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); break; @@ -1250,8 +1257,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; - if (mapMultiArgs.count("-uacomment")) { - BOOST_FOREACH(std::string cmt, mapMultiArgs.at("-uacomment")) + if (gArgs.IsArgSet("-uacomment")) { + BOOST_FOREACH(std::string cmt, gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); @@ -1264,9 +1271,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } - if (mapMultiArgs.count("-onlynet")) { + if (gArgs.IsArgSet("-onlynet")) { std::set<enum Network> nets; - BOOST_FOREACH(const std::string& snet, mapMultiArgs.at("-onlynet")) { + BOOST_FOREACH(const std::string& snet, gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); @@ -1279,8 +1286,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-whitelist")) { - BOOST_FOREACH(const std::string& net, mapMultiArgs.at("-whitelist")) { + if (gArgs.IsArgSet("-whitelist")) { + BOOST_FOREACH(const std::string& net, gArgs.GetArgs("-whitelist")) { CSubNet subnet; LookupSubNet(net.c_str(), subnet); if (!subnet.IsValid()) @@ -1341,16 +1348,16 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (fListen) { bool fBound = false; - if (mapMultiArgs.count("-bind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-bind")) { + if (gArgs.IsArgSet("-bind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) return InitError(ResolveErrMsg("bind", strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR)); } } - if (mapMultiArgs.count("-whitebind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-whitebind")) { + if (gArgs.IsArgSet("-whitebind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-whitebind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, 0, false)) return InitError(ResolveErrMsg("whitebind", strBind)); @@ -1359,7 +1366,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR | BF_WHITELIST)); } } - if (!mapMultiArgs.count("-bind") && !mapMultiArgs.count("-whitebind")) { + if (!gArgs.IsArgSet("-bind") && !gArgs.IsArgSet("-whitebind")) { struct in_addr inaddr_any; inaddr_any.s_addr = INADDR_ANY; fBound |= Bind(connman, CService(in6addr_any, GetListenPort()), BF_NONE); @@ -1369,8 +1376,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) return InitError(_("Failed to listen on any port. Use -listen=0 if you want this.")); } - if (mapMultiArgs.count("-externalip")) { - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-externalip")) { + if (gArgs.IsArgSet("-externalip")) { + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); @@ -1379,11 +1386,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-seednode")) { - BOOST_FOREACH(const std::string& strDest, mapMultiArgs.at("-seednode")) - connman.AddOneShot(strDest); - } - #if ENABLE_ZMQ pzmqNotificationInterface = CZMQNotificationInterface::Create(); @@ -1448,6 +1450,12 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); + } else { + // If necessary, upgrade from older database format. + if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; + } } if (!LoadBlockIndex(chainparams)) { @@ -1606,9 +1614,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); std::vector<fs::path> vImportFiles; - if (mapMultiArgs.count("-loadblock")) + if (gArgs.IsArgSet("-loadblock")) { - BOOST_FOREACH(const std::string& strFile, mapMultiArgs.at("-loadblock")) + BOOST_FOREACH(const std::string& strFile, gArgs.GetArgs("-loadblock")) vImportFiles.push_back(strFile); } @@ -1652,6 +1660,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; + if (gArgs.IsArgSet("-seednode")) { + connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); + } + if (!connman.Start(scheduler, strNodeError, connOptions)) return InitError(strNodeError); diff --git a/src/miner.cpp b/src/miner.cpp index 69a89bd617..28b6f23d56 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -10,11 +10,13 @@ #include "chainparams.h" #include "coins.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/merkle.h" #include "consensus/validation.h" #include "hash.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "pow.h" #include "primitives/transaction.h" diff --git a/src/net.cpp b/src/net.cpp index ed4c752606..378ac99d66 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -148,7 +148,7 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn // one by discovery. CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) { - CAddress ret(CService(CNetAddr(),GetListenPort()), NODE_NONE); + CAddress ret(CService(CNetAddr(),GetListenPort()), nLocalServices); CService addr; if (GetLocal(addr, paddrPeer)) { @@ -948,7 +948,7 @@ bool CConnman::AttemptToEvictConnection() continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, (node->nServices & nRelevantServices) == nRelevantServices, node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; @@ -1071,12 +1071,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // According to the internet TCP_NODELAY is not carried into accepted sockets // on all platforms. Set it again here just to be sure. - int set = 1; -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); if (IsBanned(addr) && !whitelisted) { @@ -1374,7 +1369,7 @@ void CConnman::ThreadSocketHandler() { if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) { - LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->id); + LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->GetId()); pnode->fDisconnect = true; } else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) @@ -1394,7 +1389,7 @@ void CConnman::ThreadSocketHandler() } else if (!pnode->fSuccessfullyConnected) { - LogPrintf("version handshake timeout from %d\n", pnode->id); + LogPrintf("version handshake timeout from %d\n", pnode->GetId()); pnode->fDisconnect = true; } } @@ -1670,12 +1665,12 @@ void CConnman::ProcessOneShot() void CConnman::ThreadOpenConnections() { // Connect to specific addresses - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) + if (gArgs.IsArgSet("-connect") && gArgs.GetArgs("-connect").size() > 0) { for (int64_t nLoop = 0;; nLoop++) { ProcessOneShot(); - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-connect")) + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-connect")) { CAddress addr(CService(), NODE_NONE); OpenNetworkConnection(addr, false, NULL, strAddr.c_str()); @@ -1726,11 +1721,17 @@ void CConnman::ThreadOpenConnections() // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; + int nOutboundRelevant = 0; std::set<std::vector<unsigned char> > setConnected; { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { if (!pnode->fInbound && !pnode->fAddnode) { + + // Count the peers that have all relevant services + if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { + nOutboundRelevant++; + } // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1748,9 +1749,9 @@ void CConnman::ThreadOpenConnections() // * Increase the number of connectable addresses in the tried table. // // Method: - // * Choose a random address from new and attempt to connect to it if we can connect + // * Choose a random address from new and attempt to connect to it if we can connect // successfully it is added to tried. - // * Start attempting feeler connections only after node finishes making outbound + // * Start attempting feeler connections only after node finishes making outbound // connections. // * Only make a feeler connection once every few minutes. // @@ -1794,14 +1795,27 @@ void CConnman::ThreadOpenConnections() continue; // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - if ((addr.nServices & nRelevantServices) != nRelevantServices && (nTries < 40 || nOutbound >= (nMaxOutbound >> 1))) + ServiceFlags nRequiredServices = nRelevantServices; + if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { + nRequiredServices = REQUIRED_SERVICES; + } + + if ((addr.nServices & nRequiredServices) != nRequiredServices) { continue; + } // do not allow non-default ports, unless after 50 invalid addresses selected already if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) continue; addrConnect = addr; + + // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services + if (nOutboundRelevant >= (nMaxOutbound >> 1)) { + addrConnect.nServices = REQUIRED_SERVICES; + } else { + addrConnect.nServices = nRequiredServices; + } break; } @@ -1877,8 +1891,8 @@ void CConnman::ThreadOpenAddedConnections() { { LOCK(cs_vAddedNodes); - if (mapMultiArgs.count("-addnode")) - vAddedNodes = mapMultiArgs.at("-addnode"); + if (gArgs.IsArgSet("-addnode")) + vAddedNodes = gArgs.GetArgs("-addnode"); } while (true) @@ -2217,6 +2231,10 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c SetBestHeight(connOptions.nBestHeight); + for (const auto& strDest : connOptions.vSeedNodes) { + AddOneShot(strDest); + } + clientInterface = connOptions.uiInterface; if (clientInterface) { clientInterface->InitMessage(_("Loading P2P addresses...")); @@ -2289,7 +2307,7 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c threadOpenAddedConnections = std::thread(&TraceThread<std::function<void()> >, "addcon", std::function<void()>(std::bind(&CConnman::ThreadOpenAddedConnections, this))); // Initiate outbound connections unless connect=0 - if (!mapMultiArgs.count("-connect") || mapMultiArgs.at("-connect").size() != 1 || mapMultiArgs.at("-connect")[0] != "0") + if (!gArgs.IsArgSet("-connect") || gArgs.GetArgs("-connect").size() != 1 || gArgs.GetArgs("-connect")[0] != "0") threadOpenConnections = std::thread(&TraceThread<std::function<void()> >, "opencon", std::function<void()>(std::bind(&CConnman::ThreadOpenConnections, this))); // Process messages @@ -2487,7 +2505,7 @@ bool CConnman::DisconnectNode(NodeId id) { LOCK(cs_vNodes); for(CNode* pnode : vNodes) { - if (id == pnode->id) { + if (id == pnode->GetId()) { pnode->fDisconnect = true; return true; } @@ -2625,10 +2643,10 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), fInbound(fInboundIn), - id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001), + id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn), @@ -2744,7 +2762,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->GetId()); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -2782,7 +2800,7 @@ bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) CNode* found = nullptr; LOCK(cs_vNodes); for (auto&& pnode : vNodes) { - if(pnode->id == id) { + if(pnode->GetId() == id) { found = pnode; break; } @@ -14,6 +14,7 @@ #include "hash.h" #include "limitedmap.h" #include "netaddress.h" +#include "policy/feerate.h" #include "protocol.h" #include "random.h" #include "streams.h" @@ -143,6 +144,7 @@ public: unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; uint64_t nMaxOutboundLimit = 0; + std::vector<std::string> vSeedNodes; }; CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); @@ -232,8 +234,6 @@ public: void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); - void AddOneShot(const std::string& strDest); - bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); @@ -291,6 +291,7 @@ private: }; void ThreadOpenAddedConnections(); + void AddOneShot(const std::string& strDest); void ProcessOneShot(); void ThreadOpenConnections(); void ThreadMessageHandler(); @@ -611,7 +612,6 @@ public: CCriticalSection cs_filter; CBloomFilter* pfilter; std::atomic<int> nRefCount; - const NodeId id; const uint64_t nKeyedNetGroup; std::atomic_bool fPauseRecv; @@ -682,6 +682,7 @@ public: private: CNode(const CNode&); void operator=(const CNode&); + const NodeId id; const uint64_t nLocalHostNonce; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 718a7de031..971db3427c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -303,6 +303,7 @@ void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); } + LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } // Requires cs_main. @@ -333,7 +334,7 @@ bool MarkBlockAsReceived(const uint256& hash) { // Requires cs_main. // returns false, still setting pit, if the block was already in flight from the same peer // pit will only be valid as long as the same cs_main lock is being held -bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); @@ -753,8 +754,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb const CTransaction& tx = *ptx; // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + for (const auto& txin : tx.vin) { + auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { const CTransaction& orphanTx = *(*mi)->second.tx; @@ -815,7 +816,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", - hashBlock.ToString(), pnode->id); + hashBlock.ToString(), pnode->GetId()); connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; } @@ -910,12 +911,11 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -950,7 +950,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { if (pnode->nVersion >= CADDR_TIME_VERSION) { - uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); @@ -1162,7 +1162,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } -uint32_t GetFetchFlags(CNode* pfrom, const CBlockIndex* pprev, const Consensus::Params& chainparams) { +uint32_t GetFetchFlags(CNode* pfrom) { uint32_t nFetchFlags = 0; if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; @@ -1176,7 +1176,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac if (req.indexes[i] >= block.vtx.size()) { LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->GetId()); return; } resp.txn[i] = block.vtx[req.indexes[i]]; @@ -1189,7 +1189,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic<bool>& interruptMsgProc) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (IsArgSet("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1268,7 +1268,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nServicesExpected & ~nServices) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); pfrom->fDisconnect = true; @@ -1278,7 +1278,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, nVersion); + LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; @@ -1380,7 +1380,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom->nVersion, - pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, + pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -1550,19 +1550,15 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); - std::vector<CInv> vToFetch; - - for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) + for (CInv &inv : vInv) { - CInv &inv = vInv[nInv]; - if (interruptMsgProc) return true; bool fAlreadyHave = AlreadyHave(inv); - LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->GetId()); if (inv.type == MSG_TX) { inv.type |= nFetchFlags; @@ -1577,14 +1573,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); } } else { pfrom->AddInventoryKnown(inv); if (fBlocksOnly) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->GetId()); } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { pfrom->AskFor(inv); } @@ -1593,9 +1589,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Track requests for our stuff GetMainSignals().Inventory(inv.hash); } - - if (!vToFetch.empty()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); } @@ -1610,10 +1603,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return error("message getdata size() = %u", vInv.size()); } - LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); + LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->GetId()); if (vInv.size() > 0) { - LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->GetId()); } pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); @@ -1653,7 +1646,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); + LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) @@ -1703,7 +1696,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); return true; } @@ -1715,7 +1708,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // might maliciously send lots of getblocktxn requests to trigger // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. - LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->GetId(), MAX_BLOCKTXN_DEPTH); CInv inv; inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; @@ -1740,7 +1733,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->GetId()); return true; } @@ -1765,7 +1758,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; int nLimit = MAX_HEADERS_RESULTS; - LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -1795,7 +1788,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { - LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->GetId()); return true; } @@ -1828,7 +1821,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->nLastTXTime = GetTime(); LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", - pfrom->id, + pfrom->GetId(), tx.GetHash().ToString(), mempool.size(), mempool.DynamicMemoryUsage() / 1000); @@ -1903,7 +1896,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } if (!fRejectedParents) { - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); BOOST_FOREACH(const CTxIn& txin, tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); pfrom->AddInventoryKnown(_inv); @@ -1948,10 +1941,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // case. int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); RelayTransaction(tx, connman); } else { - LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } } } @@ -1963,7 +1956,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (state.IsInvalid(nDoS)) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), - pfrom->id, + pfrom->GetId(), FormatStateMessage(state)); if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), @@ -2000,7 +1993,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); return true; } } @@ -2040,7 +2033,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; @@ -2064,7 +2057,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = NULL; - if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); else { @@ -2079,12 +2072,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2127,7 +2120,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { @@ -2184,7 +2177,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || it->second.first != pfrom->GetId()) { - LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->GetId()); return true; } @@ -2193,12 +2186,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received @@ -2281,7 +2274,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), pindexBestHeader->nHeight, - pfrom->id, nodestate->nUnconnectingHeaders); + pfrom->GetId(), nodestate->nUnconnectingHeaders); // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2319,7 +2312,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); CNodeState *nodestate = State(pfrom->GetId()); if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); } nodestate->nUnconnectingHeaders = 0; @@ -2330,7 +2323,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } @@ -2366,11 +2359,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Can't download any more from this peer break; } - uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->id); + pindex->GetBlockHash().ToString(), pfrom->GetId()); } if (vGetData.size() > 1) { LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", @@ -2393,7 +2386,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; - LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. @@ -2425,14 +2418,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. if (!pfrom->fInbound) { - LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->GetId()); return true; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (pfrom->fSentAddr) { - LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->GetId()); return true; } pfrom->fSentAddr = true; @@ -2533,7 +2526,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!(sProblem.empty())) { LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", - pfrom->id, + pfrom->GetId(), sProblem, pfrom->nPingNonceSent, nonce, @@ -2610,7 +2603,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(pfrom->cs_feeFilter); pfrom->minFeeFilter = newFeeFilter; } - LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->GetId()); } } @@ -2621,7 +2614,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); } @@ -2701,7 +2694,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i msg.SetVersion(pfrom->GetRecvVersion()); // Scan for message start if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); pfrom->fDisconnect = true; return false; } @@ -2710,7 +2703,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i CMessageHeader& hdr = msg.hdr; if (!hdr.IsValid(chainparams.MessageStart())) { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); return fMoreWork; } std::string strCommand = hdr.GetCommand(); @@ -2770,7 +2763,7 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i } if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); } LOCK(cs_main); @@ -2899,7 +2892,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -2929,7 +2922,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = NULL; // last header queued for delivery - ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date + ProcessBlockAvailability(pto->GetId()); // ensure pindexBestKnownBlock is up-to-date if (!fRevertToInv) { bool fFoundStartingHeader = false; @@ -2984,7 +2977,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // We only send up to 1 block as header-and-ids, as otherwise // probably means we're doing an initial-ish-sync or they're slow LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + vHeaders.front().GetHash().ToString(), pto->GetId()); int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; @@ -3014,10 +3007,10 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr LogPrint(BCLog::NET, "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, vHeaders.size(), vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); + vHeaders.back().GetHash().ToString(), pto->GetId()); } else { LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + vHeaders.front().GetHash().ToString(), pto->GetId()); } connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; @@ -3046,7 +3039,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, - pto->id, hashToAnnounce.ToString()); + pto->GetId(), hashToAnnounce.ToString()); } } } @@ -3193,7 +3186,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); pto->fDisconnect = true; return true; } @@ -3206,7 +3199,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->GetId()); pto->fDisconnect = true; return true; } @@ -3221,11 +3214,11 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); BOOST_FOREACH(const CBlockIndex *pindex, vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + uint32_t nFetchFlags = GetFetchFlags(pto); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); + pindex->nHeight, pto->GetId()); } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { @@ -3243,7 +3236,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) { - LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->id); + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); vGetData.push_back(inv); if (vGetData.size() >= 1000) { diff --git a/src/netbase.cpp b/src/netbase.cpp index bdc725359a..32557dd179 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -428,18 +428,14 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe if (hSocket == INVALID_SOCKET) return false; - int set = 1; #ifdef SO_NOSIGPIPE + int set = 1; // Different way of disabling SIGPIPE on BSD setsockopt(hSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)); #endif //Disable Nagle's algorithm -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) @@ -672,13 +668,14 @@ std::string NetworkErrorString(int err) std::string NetworkErrorString(int err) { char buf[256]; - const char *s = buf; buf[0] = 0; /* Too bad there are two incompatible implementations of the * thread-safe strerror. */ + const char *s; #ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ s = strerror_r(err, buf, sizeof(buf)); #else /* POSIX variant always returns message in buffer */ + s = buf; if (strerror_r(err, buf, sizeof(buf))) buf[0] = 0; #endif @@ -728,6 +725,13 @@ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking) return true; } +bool SetSocketNoDelay(SOCKET& hSocket) +{ + int set = 1; + int rc = setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); + return rc == 0; +} + void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index dd33b6e47e..c9d108aadd 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -59,6 +59,8 @@ std::string NetworkErrorString(int err); bool CloseSocket(SOCKET& hSocket); /** Disable or enable blocking-mode for a socket */ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking); +/** Set the TCP_NODELAY flag on a socket */ +bool SetSocketNoDelay(SOCKET& hSocket); /** * Convert milliseconds to a struct timeval for e.g. select. */ diff --git a/src/amount.cpp b/src/policy/feerate.cpp index a5f6bc3cd9..a089c02284 100644 --- a/src/amount.cpp +++ b/src/policy/feerate.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" +#include "feerate.h" #include "tinyformat.h" diff --git a/src/policy/feerate.h b/src/policy/feerate.h new file mode 100644 index 0000000000..e82268b095 --- /dev/null +++ b/src/policy/feerate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_FEERATE_H +#define BITCOIN_POLICY_FEERATE_H + +#include "amount.h" +#include "serialize.h" + +#include <string> + +extern const std::string CURRENCY_UNIT; + +/** + * Fee rate in satoshis per kilobyte: CAmount / kB + */ +class CFeeRate +{ +private: + CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes +public: + /** Fee rate of 0 satoshis per kB */ + CFeeRate() : nSatoshisPerK(0) { } + explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } + /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ + CFeeRate(const CAmount& nFeePaid, size_t nBytes); + CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } + /** + * Return the fee in satoshis for the given size in bytes. + */ + CAmount GetFee(size_t nBytes) const; + /** + * Return the fee in satoshis for a size of 1000 bytes + */ + CAmount GetFeePerK() const { return GetFee(1000); } + friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } + friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } + friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } + friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } + friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } + std::string ToString() const; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(nSatoshisPerK); + } +}; + +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index bd169f875a..66083fe1ee 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -14,6 +14,8 @@ #include "txmempool.h" #include "util.h" +static constexpr double INF_FEERATE = 1e99; + /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to their @@ -26,40 +28,43 @@ class TxConfirmStats { private: //Define the buckets we will group transactions into - std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) - std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + const std::vector<double>& buckets; // The upper-bound of the range for the bucket (inclusive) + const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket // For each bucket X: // Count the total # of txs in each bucket // Track the historical moving average of this total over blocks std::vector<double> txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector<int> curBlockTxCt; // Count the total # of txs confirmed within Y blocks in each bucket // Track the historical moving average of theses totals over blocks - std::vector<std::vector<double> > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] + std::vector<std::vector<double>> confAvg; // confAvg[Y][X] + + // Track moving avg of txs which have been evicted from the mempool + // after failing to be confirmed within Y blocks + std::vector<std::vector<double>> failAvg; // failAvg[Y][X] // Sum the total feerate of all tx's in each bucket // Track the historical moving average of this total over blocks std::vector<double> avg; - // and calculate the total for the current block to update the moving average - std::vector<double> curBlockVal; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X // Combine the total value with the tx counts to calculate the avg feerate per bucket double decay; + // Resolution (# of blocks) with which confirmations are tracked + unsigned int scale; + // Mempool counts of outstanding transactions // For each bucket X, track the number of transactions in the mempool // that are unconfirmed for each possible confirmation value Y std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket + // transactions still unconfirmed after GetMaxConfirms for each bucket std::vector<int> oldUnconfTxs; + void resizeInMemoryCounters(size_t newbuckets); + public: /** * Create new TxConfirmStats. This is called by BlockPolicyEstimator's @@ -68,9 +73,10 @@ public: * @param maxConfirms max number of confirms to track * @param decay how much to decay the historical moving average per block */ - TxConfirmStats(const std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); + TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double decay, unsigned int scale); - /** Clear the state of the curBlock variables to start counting for the new block */ + /** Roll the circular buffer for unconfirmed txs*/ void ClearCurrent(unsigned int nBlockHeight); /** @@ -86,7 +92,7 @@ public: /** Remove a transaction from mempool tracking stats*/ void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); + unsigned int bucketIndex, bool inBlock); /** Update our estimates by decaying our historical moving average and updating with the data gathered from the current block */ @@ -104,10 +110,11 @@ public: * @param nBlockHeight the current block height */ double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight) const; + double minSuccess, bool requireGreater, unsigned int nBlockHeight, + EstimationResult *result = nullptr) const; /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() const { return confAvg.size(); } + unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } /** Write state of estimation data to a file*/ void Write(CAutoFile& fileout) const; @@ -116,44 +123,47 @@ public: * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(CAutoFile& filein); + void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); }; TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay) + const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double _decay, unsigned int _scale) + : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; - for (unsigned int i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { + scale = _scale; + confAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); + } + failAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { + failAvg[i].resize(buckets.size()); } - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); + + resizeInMemoryCounters(buckets.size()); } -// Zero out the data for the current block +void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) { + // newbuckets must be passed in because the buckets referred to during Read have not been updated yet. + unconfTxs.resize(GetMaxConfirms()); + for (unsigned int i = 0; i < unconfTxs.size(); i++) { + unconfTxs[i].resize(newbuckets); + } + oldUnconfTxs.resize(newbuckets); +} + +// Roll the unconfirmed txs circular buffer void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (unsigned int j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; - for (unsigned int i = 0; i < curBlockConf.size(); i++) - curBlockConf[i][j] = 0; - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; } } @@ -163,33 +173,38 @@ void TxConfirmStats::Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + scale - 1)/scale; unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; + for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) { + confAvg[i - 1][bucketindex]++; } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; + txCtAvg[bucketindex]++; + avg[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + confAvg[i][j] = confAvg[i][j] * decay; + for (unsigned int i = 0; i < failAvg.size(); i++) + failAvg[i][j] = failAvg[i][j] * decay; + avg[j] = avg[j] * decay; + txCtAvg[j] = txCtAvg[j] * decay; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) const + unsigned int nBlockHeight, EstimationResult *result) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + scale - 1)/scale; int maxbucketindex = buckets.size() - 1; @@ -212,12 +227,21 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket; + EstimatorBucket failBucket; // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; + nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; + failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; @@ -226,24 +250,41 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { + if (passing == true) { + // First time we hit a failure record the failed bucket + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters else { + failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; + passing = true; + passBucket.withinTarget = nConf; nConf = 0; + passBucket.totalConfirmed = totalNum; totalNum = 0; + passBucket.inMempool = extraNum; + passBucket.leftMempool = failNum; + failNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + newBucketRange = true; } } } @@ -255,8 +296,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int minBucket = std::min(bestNearBucket, bestFarBucket); + unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket); for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } @@ -270,83 +311,109 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, break; } } + + passBucket.start = minBucket ? buckets[minBucket-1] : 0; + passBucket.end = buckets[maxBucket]; } - LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], - 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) { + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + } + LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d %s%.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + confTarget, requireGreater ? ">" : "<", 100.0 * successBreakPoint, decay, + median, passBucket.start, passBucket.end, + 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool), + passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool, + failBucket.start, failBucket.end, + 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool), + failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool); + + + if (result) { + result->pass = passBucket; + result->fail = failBucket; + result->decay = decay; + result->scale = scale; + } return median; } void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; - fileout << buckets; + fileout << scale; fileout << avg; fileout << txCtAvg; fileout << confAvg; + fileout << failAvg; } -void TxConfirmStats::Read(CAutoFile& filein) +void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) { - // Read data file into temporary variables and do some very basic sanity checking - std::vector<double> fileBuckets; - std::vector<double> fileAvg; - std::vector<std::vector<double> > fileConfAvg; - std::vector<double> fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - filein >> fileAvg; - if (fileAvg.size() != numBuckets) + // Read data file and do some very basic sanity checking + // buckets and bucketMap are not updated yet, so don't access them + // If there is a read failure, we'll just discard this entire object anyway + size_t maxConfirms, maxPeriods; + + // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor + if (nFileVersion >= 149900) { + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + } + + filein >> avg; + if (avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) + } + filein >> txCtAvg; + if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week - throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); + filein >> confAvg; + maxPeriods = confAvg.size(); + maxConfirms = scale * maxPeriods; - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (confAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); + if (nFileVersion >= 149900) { + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + } else { + failAvg.resize(confAvg.size()); + for (unsigned int i = 0; i < failAvg.size(); i++) { + failAvg[i].resize(numBuckets); + } } - oldUnconfTxs.resize(buckets.size()); - for (unsigned int i = 0; i < buckets.size(); i++) - bucketMap[buckets[i]] = i; + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + resizeInMemoryCounters(numBuckets); LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); @@ -360,7 +427,7 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) return bucketindex; } -void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; @@ -388,6 +455,12 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe blockIndex, bucketindex); } } + if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + unsigned int periodsAgo = blocksAgo / scale; + for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { + failAvg[i][bucketindex]++; + } + } } // This function is called from CTxMemPool::removeUnchecked to ensure @@ -395,12 +468,14 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // tracked. Txs that were part of a block have already been removed in // processBlockTx to ensure they are never double tracked, but it is // of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) +bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { LOCK(cs_feeEstimator); std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); mapMemPoolTxs.erase(hash); return true; } else { @@ -409,21 +484,28 @@ bool CBlockPolicyEstimator::removeTx(uint256 hash) } CBlockPolicyEstimator::CBlockPolicyEstimator() - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) + : nBestSeenHeight(0), firstRecordedHeight(0), historicalFirst(0), historicalBest(0), trackedTxs(0), untrackedTxs(0) { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); - minTrackedFee = CFeeRate(MIN_BUCKET_FEERATE); - std::vector<double> vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); + size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { + buckets.push_back(bucketBoundary); + bucketMap[bucketBoundary] = bucketIndex; } - vfeelist.push_back(INF_FEERATE); - feeStats = new TxConfirmStats(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + buckets.push_back(INF_FEERATE); + bucketMap[INF_FEERATE] = bucketIndex; + assert(bucketMap.size() == buckets.size()); + + feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); + shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); + longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); } CBlockPolicyEstimator::~CBlockPolicyEstimator() { delete feeStats; + delete shortStats; + delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) @@ -457,12 +539,17 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = bucketIndex; + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex2); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex3); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash())) { + if (!removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -482,6 +569,8 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } @@ -503,21 +592,32 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, // of unconfirmed txs to remove from tracking. nBestSeenHeight = nBlockHeight; - // Clear the current block state and update unconfirmed circular buffer + // Update unconfirmed circular buffer feeStats->ClearCurrent(nBlockHeight); + shortStats->ClearCurrent(nBlockHeight); + longStats->ClearCurrent(nBlockHeight); + + // Decay all exponential averages + feeStats->UpdateMovingAverages(); + shortStats->UpdateMovingAverages(); + longStats->UpdateMovingAverages(); unsigned int countedTxs = 0; - // Repopulate the current block states - for (unsigned int i = 0; i < entries.size(); i++) { - if (processBlockTx(nBlockHeight, entries[i])) + // Update averages with data points from current block + for (const auto& entry : entries) { + if (processBlockTx(nBlockHeight, entry)) countedTxs++; } - // Update all exponential averages with the current block state - feeStats->UpdateMovingAverages(); + if (firstRecordedHeight == 0 && countedTxs > 0) { + firstRecordedHeight = nBestSeenHeight; + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight); + } + - LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", + countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; untrackedTxs = 0; @@ -525,13 +625,44 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget <= 1) + return CFeeRate(0); + + return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE); +} + +CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const +{ + TxConfirmStats* stats; + double sufficientTxs = SUFFICIENT_FEETXS; + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + stats = shortStats; + sufficientTxs = SUFFICIENT_TXS_SHORT; + break; + } + case FeeEstimateHorizon::MED_HALFLIFE: { + stats = feeStats; + break; + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + stats = longStats; + break; + } + default: { + return CFeeRate(0); + } + } + LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms()) + return CFeeRate(0); + if (successThreshold > 1) return CFeeRate(0); - double median = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, nBestSeenHeight, result); if (median < 0) return CFeeRate(0); @@ -539,31 +670,148 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const +unsigned int CBlockPolicyEstimator::BlockSpan() const +{ + if (firstRecordedHeight == 0) return 0; + assert(nBestSeenHeight >= firstRecordedHeight); + + return nBestSeenHeight - firstRecordedHeight; +} + +unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const +{ + if (historicalFirst == 0) return 0; + assert(historicalBest >= historicalFirst); + + if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0; + + return historicalBest - historicalFirst; +} + +unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const +{ + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2); +} + +/** Return a fee estimate at the required successThreshold from the shortest + * time horizon which tracks confirmations up to the desired target. If + * checkShorterHorizon is requested, also allow short time horizon estimates + * for a lower target to reduce the given answer */ +double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const +{ + double estimate = -1; + if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) { + // Find estimate from shortest time horizon possible + if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon + estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + } + else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon + estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + else { // long horizon + estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + if (checkShorterHorizon) { + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > feeStats->GetMaxConfirms()) { + double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + estimate = medMax; + } + if (confTarget > shortStats->GetMaxConfirms()) { + double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + estimate = shortMax; + } + } + } + return estimate; +} + +/** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + * at 2 * target for any longer time horizons. + */ +double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget) const +{ + double estimate = -1; + if (doubleTarget <= shortStats->GetMaxConfirms()) { + estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + } + if (doubleTarget <= feeStats->GetMaxConfirms()) { + double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + if (longEstimate > estimate) { + estimate = longEstimate; + } + } + return estimate; +} + +/** estimateSmartFee returns the max of the feerates calculated with a 60% + * threshold required at target / 2, an 85% threshold required at target and a + * 95% threshold required at 2 * target. Each calculation is performed at the + * shortest time horizon which tracks the required target. Conservative + * estimates, however, required the 95% threshold at 2 * target be met for any + * longer time horizons also. + */ +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; double median = -1; - { LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) return CFeeRate(0); // It's not possible to get reasonable estimates for confTarget of 1 if (confTarget == 1) confTarget = 2; - while (median < 0 && (unsigned int)confTarget <= feeStats->GetMaxConfirms()) { - median = feeStats->EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + unsigned int maxUsableEstimate = MaxUsableEstimate(); + if (maxUsableEstimate <= 1) + return CFeeRate(0); + + if ((unsigned int)confTarget > maxUsableEstimate) { + confTarget = maxUsableEstimate; + } + + assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + /** true is passed to estimateCombined fee for target/2 and target so + * that we check the max confirms for shorter time horizons as well. + * This is necessary to preserve monotonically increasing estimates. + * For non-conservative estimates we do the same thing for 2*target, but + * for conservative estimates we want to skip these shorter horizons + * checks for 2*target because we are taking the max over all time + * horizons so we already have monotonically increasing estimates and + * the purpose of conservative estimates is not to let short term + * fluctuations lower our estimates by too much. + */ + double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true); + double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true); + double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative); + median = halfEst; + if (actualEst > median) { + median = actualEst; + } + if (doubleEst > median) { + median = doubleEst; + } + + if (conservative || median == -1) { + double consEst = estimateConservativeFee(2 * confTarget); + if (consEst > median) { + median = consEst; + } } } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; + *answerFoundAtTarget = confTarget; // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); @@ -576,14 +824,24 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } + bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { try { LOCK(cs_feeEstimator); - fileout << 139900; // version required to read: 0.13.99 or later + fileout << 149900; // version required to read: 0.14.99 or later fileout << CLIENT_VERSION; // version that wrote the file fileout << nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()/2) { + fileout << firstRecordedHeight << nBestSeenHeight; + } + else { + fileout << historicalFirst << historicalBest; + } + fileout << buckets; feeStats->Write(fileout); + shortStats->Write(fileout); + longStats->Write(fileout); } catch (const std::exception&) { LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); @@ -596,27 +854,104 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) { try { LOCK(cs_feeEstimator); - int nVersionRequired, nVersionThatWrote, nFileBestSeenHeight; + int nVersionRequired, nVersionThatWrote; + unsigned int nFileBestSeenHeight, nFileHistoricalFirst, nFileHistoricalBest; filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + + // Read fee estimates file into temporary variables so existing data + // structures aren't corrupted if there is an exception. filein >> nFileBestSeenHeight; - feeStats->Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + + if (nVersionThatWrote < 149900) { + // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. + // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor + double tempDecay; + filein >> tempDecay; + if (tempDecay <= 0 || tempDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + std::vector<double> tempBuckets; + filein >> tempBuckets; + size_t tempNum = tempBuckets.size(); + if (tempNum <= 1 || tempNum > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::map<double, unsigned int> tempMap; + + std::unique_ptr<TxConfirmStats> tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); + tempFeeStats->Read(filein, nVersionThatWrote, tempNum); + // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + + tempMap.clear(); + for (unsigned int i = 0; i < tempBuckets.size(); i++) { + tempMap[tempBuckets[i]] = i; + } + } + else { // nVersionThatWrote >= 149900 + filein >> nFileHistoricalFirst >> nFileHistoricalBest; + if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { + throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); + } + std::vector<double> fileBuckets; + filein >> fileBuckets; + size_t numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); + fileShortStats->Read(filein, nVersionThatWrote, numBuckets); + fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + + // Fee estimates file parsed correctly + // Copy buckets from file and refresh our bucketmap + buckets = fileBuckets; + bucketMap.clear(); + for (unsigned int i = 0; i < buckets.size(); i++) { + bucketMap[buckets[i]] = i; + } + + // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap + delete feeStats; + delete shortStats; + delete longStats; + feeStats = fileFeeStats.release(); + shortStats = fileShortStats.release(); + longStats = fileLongStats.release(); + + nBestSeenHeight = nFileBestSeenHeight; + historicalFirst = nFileHistoricalFirst; + historicalBest = nFileHistoricalBest; + } } - catch (const std::exception&) { - LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal)\n"); + catch (const std::exception& e) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); return false; } return true; } +void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { + int64_t startclear = GetTimeMicros(); + std::vector<uint256> txids; + pool.queryHashes(txids); + LOCK(cs_feeEstimator); + for (auto& txid : txids) { + removeTx(txid, false); + } + int64_t endclear = GetTimeMicros(); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %ld micros\n",txids.size(), endclear - startclear); +} + FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) { CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { + for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { feeset.insert(bucketBoundary); } } diff --git a/src/policy/fees.h b/src/policy/fees.h index 34f07c7270..e99fec2c39 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -6,6 +6,7 @@ #define BITCOIN_POLICYESTIMATOR_H #include "amount.h" +#include "feerate.h" #include "uint256.h" #include "random.h" #include "sync.h" @@ -41,53 +42,57 @@ class TxConfirmStats; * within your desired 5 blocks. * * Here is a brief description of the implementation: - * When a transaction enters the mempool, we - * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of feerate - * paid in each bucket. Then we calculate how many blocks Y it took each - * transaction to be mined and we track an array of counters in each bucket - * for how long it to took transactions to get confirmed from 1 to a max of 25 - * and we increment all the counters from Y up to 25. This is because for any - * number Z>=Y the transaction was successfully mined within Z blocks. We - * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given feerate - * bucket and the total number that were confirmed in each number 1-25 blocks - * or less for any bucket. We save this history by keeping an exponentially - * decaying moving average of each one of these stats. Furthermore we also - * keep track of the number unmined (in mempool) transactions in each bucket - * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that feerate bucket when calculating - * an estimate for any number of confirmations below the number of blocks - * they've been outstanding. + * When a transaction enters the mempool, we track the height of the block chain + * at entry. All further calculations are conducted only on this set of "seen" + * transactions. Whenever a block comes in, we count the number of transactions + * in each bucket and the total amount of feerate paid in each bucket. Then we + * calculate how many blocks Y it took each transaction to be mined. We convert + * from a number of blocks to a number of periods Y' each encompassing "scale" + * blocks. This is is tracked in 3 different data sets each up to a maximum + * number of periods. Within each data set we have an array of counters in each + * feerate bucket and we increment all the counters from Y' up to max periods + * representing that a tx was successfully confirmed in less than or equal to + * that many periods. We want to save a history of this information, so at any + * time we have a counter of the total number of transactions that happened in a + * given feerate bucket and the total number that were confirmed in each of the + * periods or less for any bucket. We save this history by keeping an + * exponentially decaying moving average of each one of these stats. This is + * done for a different decay in each of the 3 data sets to keep relevant data + * from different time horizons. Furthermore we also keep track of the number + * unmined (in mempool or left mempool without being included in a block) + * transactions in each bucket and for how many blocks they have been + * outstanding and use both of these numbers to increase the number of transactions + * we've seen in that feerate bucket when calculating an estimate for any number + * of confirmations below the number of blocks they've been outstanding. */ -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ -static const double SUFFICIENT_FEETXS = 1; +/* Identifier for each of the 3 different TxConfirmStats which will track + * history over different time horizons. */ +enum FeeEstimateHorizon { + SHORT_HALFLIFE = 0, + MED_HALFLIFE = 1, + LONG_HALFLIFE = 2 +}; -// Minimum and Maximum values for tracking feerates -// The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we -// might ever want to track. Historically this has been 1000 since it was -// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it -// invalidates old estimates files. So leave it at 1000 unless it becomes -// necessary to lower it, and then lower it substantially. -static constexpr double MIN_BUCKET_FEERATE = 1000; -static const double MAX_BUCKET_FEERATE = 1e7; -static const double INF_FEERATE = MAX_MONEY; +/* Used to return detailed information about a feerate bucket */ +struct EstimatorBucket +{ + double start = -1; + double end = -1; + double withinTarget = 0; + double totalConfirmed = 0; + double inMempool = 0; + double leftMempool = 0; +}; -// We have to lump transactions into buckets based on feerate, but we want to be able -// to give accurate estimates over a large range of potential feerates -// Therefore it makes sense to exponentially space the buckets -/** Spacing of FeeRate buckets */ -static const double FEE_SPACING = 1.1; +/* Used to return detailed information about a fee estimate calculation */ +struct EstimationResult +{ + EstimatorBucket pass; + EstimatorBucket fail; + double decay; + unsigned int scale; +}; /** * We want to be able to estimate feerates that are needed on tx's to be included in @@ -96,6 +101,55 @@ static const double FEE_SPACING = 1.1; */ class CBlockPolicyEstimator { +private: + /** Track confirm delays up to 12 blocks for short horizon */ + static constexpr unsigned int SHORT_BLOCK_PERIODS = 12; + static constexpr unsigned int SHORT_SCALE = 1; + /** Track confirm delays up to 48 blocks for medium horizon */ + static constexpr unsigned int MED_BLOCK_PERIODS = 24; + static constexpr unsigned int MED_SCALE = 2; + /** Track confirm delays up to 1008 blocks for long horizon */ + static constexpr unsigned int LONG_BLOCK_PERIODS = 42; + static constexpr unsigned int LONG_SCALE = 24; + /** Historical estimates that are older than this aren't valid */ + static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; + + /** Decay of .962 is a half-life of 18 blocks or about 3 hours */ + static constexpr double SHORT_DECAY = .962; + /** Decay of .998 is a half-life of 144 blocks or about 1 day */ + static constexpr double MED_DECAY = .9952; + /** Decay of .9995 is a half-life of 1008 blocks or about 1 week */ + static constexpr double LONG_DECAY = .99931; + + /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ + static constexpr double HALF_SUCCESS_PCT = .6; + /** Require greater than 85% of X feerate transactions to be confirmed within Y blocks*/ + static constexpr double SUCCESS_PCT = .85; + /** Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks*/ + static constexpr double DOUBLE_SUCCESS_PCT = .95; + + /** Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance */ + static constexpr double SUFFICIENT_FEETXS = 0.1; + /** Require an avg of 0.5 tx when using short decay since there are fewer blocks considered*/ + static constexpr double SUFFICIENT_TXS_SHORT = 0.5; + + /** Minimum and Maximum values for tracking feerates + * The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we + * might ever want to track. Historically this has been 1000 since it was + * inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + * invalidates old estimates files. So leave it at 1000 unless it becomes + * necessary to lower it, and then lower it substantially. + */ + static constexpr double MIN_BUCKET_FEERATE = 1000; + static constexpr double MAX_BUCKET_FEERATE = 1e7; + + /** Spacing of FeeRate buckets + * We have to lump transactions into buckets based on feerate, but we want to be able + * to give accurate estimates over a large range of potential feerates + * Therefore it makes sense to exponentially space the buckets + */ + static constexpr double FEE_SPACING = 1.05; + public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); @@ -109,16 +163,23 @@ public: void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); + bool removeTx(uint256 hash, bool inBlock); - /** Return a feerate estimate */ + /** DEPRECATED. Return a feerate estimate */ CFeeRate estimateFee(int confTarget) const; - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + /** Estimate feerate needed to get be included in a block within confTarget + * blocks. If no answer can be given at confTarget, return an estimate at + * the closest target where one can be given. 'conservative' estimates are + * valid over longer time horizons also. + */ + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative = true) const; + + /** Return a specific fee estimate calculation with a given success + * threshold and time horizon, and optionally return detailed data about + * calculation */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const; + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; /** Write estimation data to a file */ bool Write(CAutoFile& fileout) const; @@ -126,9 +187,15 @@ public: /** Read estimation data from a file */ bool Read(CAutoFile& filein); + /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ + void FlushUnconfirmed(CTxMemPool& pool); + private: - CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main unsigned int nBestSeenHeight; + unsigned int firstRecordedHeight; + unsigned int historicalFirst; + unsigned int historicalBest; + struct TxStatsInfo { unsigned int blockHeight; @@ -141,19 +208,42 @@ private: /** Classes to track historical data on transaction confirmations */ TxConfirmStats* feeStats; + TxConfirmStats* shortStats; + TxConfirmStats* longStats; unsigned int trackedTxs; unsigned int untrackedTxs; + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + mutable CCriticalSection cs_feeEstimator; /** Process a transaction confirmed in a block*/ bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + /** Helper for estimateSmartFee */ + double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const; + /** Helper for estimateSmartFee */ + double estimateConservativeFee(unsigned int doubleTarget) const; + /** Number of blocks of data recorded while fee estimates have been running */ + unsigned int BlockSpan() const; + /** Number of blocks of recorded fee estimate data represented in saved data file */ + unsigned int HistoricalBlockSpan() const; + /** Calculation of highest target that reasonable estimate can be provided for */ + unsigned int MaxUsableEstimate() const; }; class FeeFilterRounder { +private: + static constexpr double MAX_FILTER_FEERATE = 1e7; + /** FEE_FILTER_SPACING is just used to provide some quantization of fee + * filter results. Historically it reused FEE_SPACING, but it is completely + * unrelated, and was made a separate constant so the two concepts are not + * tied together */ + static constexpr double FEE_FILTER_SPACING = 1.1; + public: /** Create new FeeFilterRounder */ FeeFilterRounder(const CFeeRate& minIncrementalFee); @@ -165,4 +255,5 @@ private: std::set<double> feeset; FastRandomContext insecure_rand; }; + #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 6f8b6c2953..14d58e7442 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -15,6 +15,43 @@ #include <boost/foreach.hpp> +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) +{ + // "Dust" is defined in terms of dustRelayFee, + // which has units satoshis-per-kilobyte. + // If you'd pay more than 1/3 in fees + // to spend something, then we consider it dust. + // A typical spendable non-segwit txout is 34 bytes big, and will + // need a CTxIn of at least 148 bytes to spend: + // so dust is a spendable txout less than + // 546*dustRelayFee/1000 (in satoshis). + // A typical spendable segwit txout is 31 bytes big, and will + // need a CTxIn of at least 67 bytes to spend: + // so dust is a spendable txout less than + // 294*dustRelayFee/1000 (in satoshis). + if (txout.scriptPubKey.IsUnspendable()) + return 0; + + size_t nSize = GetSerializeSize(txout, SER_DISK, 0); + int witnessversion = 0; + std::vector<unsigned char> witnessprogram; + + if (txout.scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { + // sum the sizes of the parts of a transaction input + // with 75% segwit discount applied to the script size. + nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); + } else { + nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above + } + + return 3 * dustRelayFeeIn.GetFee(nSize); +} + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) +{ + return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); +} + /** * Check transaction inputs to mitigate two * potential denial-of-service attacks: @@ -106,7 +143,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnes else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { reason = "bare-multisig"; return false; - } else if (txout.IsDust(dustRelayFee)) { + } else if (IsDust(txout, ::dustRelayFee)) { reason = "dust"; return false; } @@ -128,7 +165,7 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; txnouttype whichType; @@ -167,7 +204,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) if (tx.vin[i].scriptWitness.IsNull()) continue; - const CTxOut &prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut &prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; // get the scriptPubKey corresponding to this input: CScript prevScript = prev.scriptPubKey; diff --git a/src/policy/policy.h b/src/policy/policy.h index 6df541bc0f..2c2ea9d5b8 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -7,12 +7,14 @@ #define BITCOIN_POLICY_POLICY_H #include "consensus/consensus.h" +#include "feerate.h" #include "script/interpreter.h" #include "script/standard.h" #include <string> class CCoinsViewCache; +class CTxOut; /** Default for -blockmaxsize, which controls the maximum size of block the mining code will create **/ static const unsigned int DEFAULT_BLOCK_MAX_SIZE = 750000; @@ -72,6 +74,10 @@ static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_ static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST; +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); + bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled = false); /** * Check for standard transaction types diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 5059030309..00ac0b92b5 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -161,43 +161,6 @@ public: return (nValue == -1); } - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const - { - // "Dust" is defined in terms of CTransaction::minRelayTxFee, - // which has units satoshis-per-kilobyte. - // If you'd pay more than 1/3 in fees - // to spend something, then we consider it dust. - // A typical spendable non-segwit txout is 34 bytes big, and will - // need a CTxIn of at least 148 bytes to spend: - // so dust is a spendable txout less than - // 546*minRelayTxFee/1000 (in satoshis). - // A typical spendable segwit txout is 31 bytes big, and will - // need a CTxIn of at least 67 bytes to spend: - // so dust is a spendable txout less than - // 294*minRelayTxFee/1000 (in satoshis). - if (scriptPubKey.IsUnspendable()) - return 0; - - size_t nSize = GetSerializeSize(*this, SER_DISK, 0); - int witnessversion = 0; - std::vector<unsigned char> witnessprogram; - - if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { - // sum the sizes of the parts of a transaction input - // with 75% segwit discount applied to the script size. - nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); - } else { - nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above - } - - return 3 * minRelayTxFee.GetFee(nSize); - } - - bool IsDust(const CFeeRate &minRelayTxFee) const - { - return (nValue < GetDustThreshold(minRelayTxFee)); - } - friend bool operator==(const CTxOut& a, const CTxOut& b) { return (a.nValue == b.nValue && diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 38ad6e9aab..135cf6f701 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -434,8 +434,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) { CTxOut txout(amount, (CScript)std::vector<unsigned char>(24, 0)); txDummy.vout.push_back(txout); - if (txout.IsDust(dustRelayFee)) - fDust = true; + fDust |= IsDust(txout, ::dustRelayFee); } } @@ -514,8 +513,6 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) // Fee nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); - if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) - nPayFee = coinControl->nMinimumTotalFee; if (nPayAmount > 0) { @@ -527,10 +524,10 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (nChange > 0 && nChange < MIN_CHANGE) { CTxOut txout(nChange, (CScript)std::vector<unsigned char>(24, 0)); - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (CoinControlDialog::fSubtractFeeFromAmount) // dust-change will be raised until no dust - nChange = txout.GetDustThreshold(dustRelayFee); + nChange = GetDustThreshold(txout, ::dustRelayFee); else { nPayFee += nChange; @@ -574,7 +571,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) l5->setText(((nBytes > 0) ? ASYMP_UTF8 : "") + QString::number(nBytes)); // Bytes l7->setText(fDust ? tr("yes") : tr("no")); // Dust l8->setText(BitcoinUnits::formatWithUnit(nDisplayUnit, nChange)); // Change - if (nPayFee > 0 && (coinControl->nMinimumTotalFee < nPayFee)) + if (nPayFee > 0) { l3->setText(ASYMP_UTF8 + l3->text()); l4->setText(ASYMP_UTF8 + l4->text()); diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 52256ca5c4..89f9c25d14 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -862,19 +862,6 @@ </widget> </item> <item> - <widget class="QRadioButton" name="radioCustomAtLeast"> - <property name="toolTip"> - <string>If the custom fee is set to 1000 satoshis and the transaction is only 250 bytes, then "per kilobyte" only pays 250 satoshis in fee, while "total at least" pays 1000 satoshis. For transactions bigger than a kilobyte both pay by kilobyte.</string> - </property> - <property name="text"> - <string>total at least</string> - </property> - <attribute name="buttonGroup"> - <string notr="true">groupCustomFee</string> - </attribute> - </widget> - </item> - <item> <widget class="BitcoinAmountField" name="customFee"/> </item> <item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 3f3f9b9ccb..bffa81137b 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -251,7 +251,7 @@ bool isDust(const QString& address, const CAmount& amount) CTxDestination dest = CBitcoinAddress(address.toStdString()).Get(); CScript script = GetScriptForDestination(dest); CTxOut txOut(amount, script); - return txOut.IsDust(dustRelayFee); + return IsDust(txOut, ::dustRelayFee); } QString HtmlEscape(const QString& str, bool fMultiLine) diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index dd75f12076..c31a7a478d 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -219,14 +219,16 @@ void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) if (GUIUtil::parseBitcoinURI(arg, &r) && !r.address.isEmpty()) { CBitcoinAddress address(r.address.toStdString()); + auto tempChainParams = CreateChainParams(CBaseChainParams::MAIN); - if (address.IsValid(Params(CBaseChainParams::MAIN))) + if (address.IsValid(*tempChainParams)) { SelectParams(CBaseChainParams::MAIN); } - else if (address.IsValid(Params(CBaseChainParams::TESTNET))) - { - SelectParams(CBaseChainParams::TESTNET); + else { + tempChainParams = CreateChainParams(CBaseChainParams::TESTNET); + if (address.IsValid(*tempChainParams)) + SelectParams(CBaseChainParams::TESTNET); } } } @@ -580,7 +582,7 @@ bool PaymentServer::processPaymentRequest(const PaymentRequestPlus& request, Sen // Extract and check amounts CTxOut txOut(sendingTo.second, sendingTo.first); - if (txOut.IsDust(dustRelayFee)) { + if (IsDust(txOut, ::dustRelayFee)) { Q_EMIT message(tr("Payment request error"), tr("Requested payment amount of %1 is too small (considered dust).") .arg(BitcoinUnits::formatWithUnit(optionsModel->getDisplayUnit(), sendingTo.second)), CClientUIInterface::MSG_ERROR); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 7f2f83d9f7..b200cb1127 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -98,7 +98,7 @@ class QtRPCTimerBase: public QObject, public RPCTimerBase { Q_OBJECT public: - QtRPCTimerBase(boost::function<void(void)>& _func, int64_t millis): + QtRPCTimerBase(std::function<void(void)>& _func, int64_t millis): func(_func) { timer.setSingleShot(true); @@ -110,7 +110,7 @@ private Q_SLOTS: void timeout() { func(); } private: QTimer timer; - boost::function<void(void)> func; + std::function<void(void)> func; }; class QtRPCTimerInterface: public RPCTimerInterface @@ -118,7 +118,7 @@ class QtRPCTimerInterface: public RPCTimerInterface public: ~QtRPCTimerInterface() {} const char *Name() { return "Qt"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new QtRPCTimerBase(func, millis); } @@ -730,8 +730,14 @@ void RPCConsole::clear(bool clearHistory) ).arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize)) ); +#ifdef Q_OS_MAC + QString clsKey = "(⌘)-L"; +#else + QString clsKey = "Ctrl-L"; +#endif + message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + - tr("Use up and down arrows to navigate history, and <b>Ctrl-L</b> to clear screen.") + "<br>" + + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + tr("Type <b>help</b> for an overview of available commands.")) + "<br><span class=\"secwarning\">" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramification of a command.") + diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 098cda6d32..272ab9486a 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -31,8 +31,6 @@ #include <QTextDocument> #include <QTimer> -#define SEND_CONFIRM_DELAY 3 - SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::SendCoinsDialog), @@ -111,7 +109,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupFee->setId(ui->radioCustomFee, 1); ui->groupFee->button((int)std::max(0, std::min(1, settings.value("nFeeRadio").toInt())))->setChecked(true); ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); - ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); @@ -608,7 +605,6 @@ void SendCoinsDialog::updateFeeSectionControls() ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); - ui->radioCustomAtLeast ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() && CoinControlDialog::coinControl->HasSelected()); ui->customFee ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); } @@ -619,19 +615,12 @@ void SendCoinsDialog::updateGlobalFeeVariables() int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; payTxFee = CFeeRate(0); - // set nMinimumTotalFee to 0 to not accidentally pay a custom fee - CoinControlDialog::coinControl->nMinimumTotalFee = 0; - // show the estimated required time for confirmation ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget * Params().GetConsensus().nPowTargetSpacing) + " / " + tr("%n block(s)", "", nConfirmTarget)); } else { payTxFee = CFeeRate(ui->customFee->value()); - - // if user has selected to set a minimum absolute fee, pass the value to coincontrol - // set nMinimumTotalFee to 0 in case of user has selected that the fee is per KB - CoinControlDialog::coinControl->nMinimumTotalFee = ui->radioCustomAtLeast->isChecked() ? ui->customFee->value() : 0; } } @@ -830,21 +819,6 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - if (model->getOptionsModel()->getCoinControlFeatures()) - { - // enable minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(true); - - // only enable the feature if inputs are selected - ui->radioCustomAtLeast->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() &&CoinControlDialog::coinControl->HasSelected()); - } - else - { - // in case coin control is disabled (=default), hide minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(false); - return; - } - // set pay amounts CoinControlDialog::payAmounts.clear(); CoinControlDialog::fSubtractFeeFromAmount = false; diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index a402edc961..a932f129be 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -100,13 +100,14 @@ Q_SIGNALS: }; +#define SEND_CONFIRM_DELAY 3 class SendConfirmationDialog : public QMessageBox { Q_OBJECT public: - SendConfirmationDialog(const QString &title, const QString &text, int secDelay = 0, QWidget *parent = 0); + SendConfirmationDialog(const QString &title, const QString &text, int secDelay = SEND_CONFIRM_DELAY, QWidget *parent = 0); int exec(); private Q_SLOTS: diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index a0dce3d997..ff1eb59f16 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -8,26 +8,46 @@ #include "qt/sendcoinsdialog.h" #include "qt/sendcoinsentry.h" #include "qt/transactiontablemodel.h" +#include "qt/transactionview.h" #include "qt/walletmodel.h" #include "test/test_bitcoin.h" #include "validation.h" #include "wallet/wallet.h" #include <QAbstractButton> +#include <QAction> #include <QApplication> +#include <QCheckBox> +#include <QPushButton> #include <QTimer> #include <QVBoxLayout> namespace { -//! Press "Yes" button in modal send confirmation dialog. -void ConfirmSend() +//! Press "Ok" button in message box dialog. +void ConfirmMessage(QString* text = nullptr) { - QTimer::singleShot(0, makeCallback([](Callback* callback) { + QTimer::singleShot(0, makeCallback([text](Callback* callback) { + for (QWidget* widget : QApplication::topLevelWidgets()) { + if (widget->inherits("QMessageBox")) { + QMessageBox* messageBox = qobject_cast<QMessageBox*>(widget); + if (text) *text = messageBox->text(); + messageBox->defaultButton()->click(); + } + } + delete callback; + }), SLOT(call())); +} + +//! Press "Yes" or "Cancel" buttons in modal send confirmation dialog. +void ConfirmSend(QString* text = nullptr, bool cancel = false) +{ + QTimer::singleShot(0, makeCallback([text, cancel](Callback* callback) { for (QWidget* widget : QApplication::topLevelWidgets()) { if (widget->inherits("SendConfirmationDialog")) { SendConfirmationDialog* dialog = qobject_cast<SendConfirmationDialog*>(widget); - QAbstractButton* button = dialog->button(QMessageBox::Yes); + if (text) *text = dialog->text(); + QAbstractButton* button = dialog->button(cancel ? QMessageBox::Cancel : QMessageBox::Yes); button->setEnabled(true); button->click(); } @@ -37,12 +57,16 @@ void ConfirmSend() } //! Send coins to address and return txid. -uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount) +uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount, bool rbf) { QVBoxLayout* entries = sendCoinsDialog.findChild<QVBoxLayout*>("entries"); SendCoinsEntry* entry = qobject_cast<SendCoinsEntry*>(entries->itemAt(0)->widget()); entry->findChild<QValidatedLineEdit*>("payTo")->setText(QString::fromStdString(address.ToString())); entry->findChild<BitcoinAmountField*>("payAmount")->setValue(amount); + sendCoinsDialog.findChild<QFrame*>("frameFee") + ->findChild<QFrame*>("frameFeeSelection") + ->findChild<QCheckBox*>("optInRBF") + ->setCheckState(rbf ? Qt::Checked : Qt::Unchecked); uint256 txid; boost::signals2::scoped_connection c(wallet.NotifyTransactionChanged.connect([&txid](CWallet*, const uint256& hash, ChangeType status) { if (status == CT_NEW) txid = hash; @@ -65,6 +89,42 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) } return {}; } + +//! Request context menu (call method that is public in qt5, but protected in qt4). +void RequestContextMenu(QWidget* widget) +{ + class Qt4Hack : public QWidget + { + public: + using QWidget::customContextMenuRequested; + }; + static_cast<Qt4Hack*>(widget)->customContextMenuRequested({}); +} + +//! Invoke bumpfee on txid and check results. +void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, std::string expectError, bool cancel) +{ + QTableView* table = view.findChild<QTableView*>("transactionView"); + QModelIndex index = FindTx(*table->selectionModel()->model(), txid); + QVERIFY2(index.isValid(), "Could not find BumpFee txid"); + + // Select row in table, invoke context menu, and make sure bumpfee action is + // enabled or disabled as expected. + QAction* action = view.findChild<QAction*>("bumpFeeAction"); + table->selectionModel()->select(index, QItemSelectionModel::ClearAndSelect | QItemSelectionModel::Rows); + action->setEnabled(expectDisabled); + RequestContextMenu(table); + QCOMPARE(action->isEnabled(), !expectDisabled); + + action->setEnabled(true); + QString text; + if (expectError.empty()) { + ConfirmSend(&text, cancel); + } else { + ConfirmMessage(&text); + } + action->trigger(); + QVERIFY(text.indexOf(QString::fromStdString(expectError)) != -1); } //! Simple qt wallet tests. @@ -80,11 +140,13 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) // src/qt/test/test_bitcoin-qt -platform xcb # Linux // src/qt/test/test_bitcoin-qt -platform windows # Windows // src/qt/test/test_bitcoin-qt -platform cocoa # macOS -void WalletTests::walletTests() +void TestSendCoins() { - // Set up wallet and chain with 101 blocks (1 mature block for spending). + // Set up wallet and chain with 105 blocks (5 mature blocks for spending). TestChain100Setup test; - test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + for (int i = 0; i < 5; ++i) { + test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + } bitdb.MakeMock(); std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); CWallet wallet(std::move(dbw)); @@ -101,19 +163,34 @@ void WalletTests::walletTests() // Create widgets for sending coins and listing transactions. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); + TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel; WalletModel walletModel(platformStyle.get(), &wallet, &optionsModel); sendCoinsDialog.setModel(&walletModel); + transactionView.setModel(&walletModel); // Send two transactions, and verify they are added to transaction list. TransactionTableModel* transactionTableModel = walletModel.getTransactionTableModel(); - QCOMPARE(transactionTableModel->rowCount({}), 101); - uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN); - uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN); - QCOMPARE(transactionTableModel->rowCount({}), 103); + QCOMPARE(transactionTableModel->rowCount({}), 105); + uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN, false /* rbf */); + uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN, true /* rbf */); + QCOMPARE(transactionTableModel->rowCount({}), 107); QVERIFY(FindTx(*transactionTableModel, txid1).isValid()); QVERIFY(FindTx(*transactionTableModel, txid2).isValid()); + // Call bumpfee. Test disabled, canceled, enabled, then failing cases. + BumpFee(transactionView, txid1, true /* expect disabled */, "not BIP 125 replaceable" /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, true /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, true /* expect disabled */, "already bumped" /* expected error */, false /* cancel */); + bitdb.Flush(true); bitdb.Reset(); } + +} + +void WalletTests::walletTests() +{ + TestSendCoins(); +} diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index d81188895b..233fc08772 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -293,13 +293,12 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco { COutPoint prevout = txin.prevout; - CCoins prev; - if(pcoinsTip->GetCoins(prevout.hash, prev)) + Coin prev; + if(pcoinsTip->GetCoin(prevout, prev)) { - if (prevout.n < prev.vout.size()) { strHTML += "<li>"; - const CTxOut &vout = prev.vout[prevout.n]; + const CTxOut &vout = prev.out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 4bb260aa58..0090b0c74b 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -246,13 +246,13 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) status.status = TransactionStatus::Confirmed; } } - + status.needsUpdate = false; } bool TransactionRecord::statusUpdateNeeded() { AssertLockHeld(cs_main); - return status.cur_num_blocks != chainActive.Height(); + return status.cur_num_blocks != chainActive.Height() || status.needsUpdate; } QString TransactionRecord::getTxID() const diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index 5aabbbffa8..59f681224f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -61,6 +61,8 @@ public: /** Current number of blocks (to know whether cached status is still valid) */ int cur_num_blocks; + + bool needsUpdate; }; /** UI model for a transaction. A core transaction can be represented by multiple UI transactions if it has diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 61466c8ed1..f27abc2104 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -168,6 +168,10 @@ public: case CT_UPDATED: // Miscellaneous updates -- nothing to do, status update will take care of this, and is only computed for // visible transactions. + for (int i = lowerIndex; i < upperIndex; i++) { + TransactionRecord *rec = &cachedWallet[i]; + rec->status.needsUpdate = true; + } break; } } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 30f4db9450..e3e070b27f 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -11,6 +11,7 @@ #include "guiutil.h" #include "optionsmodel.h" #include "platformstyle.h" +#include "sendcoinsdialog.h" #include "transactiondescdialog.h" #include "transactionfilterproxy.h" #include "transactionrecord.h" @@ -37,7 +38,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), model(0), transactionProxyModel(0), - transactionView(0), abandonAction(0), columnResizingFixer(0) + transactionView(0), abandonAction(0), bumpFeeAction(0), columnResizingFixer(0) { // Build filter row setContentsMargins(0,0,0,0); @@ -135,9 +136,12 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa view->installEventFilter(this); transactionView = view; + transactionView->setObjectName("transactionView"); // Actions abandonAction = new QAction(tr("Abandon transaction"), this); + bumpFeeAction = new QAction(tr("Increase transaction fee"), this); + bumpFeeAction->setObjectName("bumpFeeAction"); QAction *copyAddressAction = new QAction(tr("Copy address"), this); QAction *copyLabelAction = new QAction(tr("Copy label"), this); QAction *copyAmountAction = new QAction(tr("Copy amount"), this); @@ -148,6 +152,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa QAction *showDetailsAction = new QAction(tr("Show transaction details"), this); contextMenu = new QMenu(this); + contextMenu->setObjectName("contextMenu"); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyAmountAction); @@ -156,6 +161,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa contextMenu->addAction(copyTxPlainText); contextMenu->addAction(showDetailsAction); contextMenu->addSeparator(); + contextMenu->addAction(bumpFeeAction); contextMenu->addAction(abandonAction); contextMenu->addAction(editLabelAction); @@ -173,6 +179,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SIGNAL(doubleClicked(QModelIndex))); connect(view, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(contextualMenu(QPoint))); + connect(bumpFeeAction, SIGNAL(triggered()), this, SLOT(bumpFee())); connect(abandonAction, SIGNAL(triggered()), this, SLOT(abandonTx())); connect(copyAddressAction, SIGNAL(triggered()), this, SLOT(copyAddress())); connect(copyLabelAction, SIGNAL(triggered()), this, SLOT(copyLabel())); @@ -372,10 +379,11 @@ void TransactionView::contextualMenu(const QPoint &point) uint256 hash; hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->transactionCanBeAbandoned(hash)); + bumpFeeAction->setEnabled(model->transactionCanBeBumped(hash)); if(index.isValid()) { - contextMenu->exec(QCursor::pos()); + contextMenu->popup(transactionView->viewport()->mapToGlobal(point)); } } @@ -397,6 +405,24 @@ void TransactionView::abandonTx() model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); } +void TransactionView::bumpFee() +{ + if(!transactionView || !transactionView->selectionModel()) + return; + QModelIndexList selection = transactionView->selectionModel()->selectedRows(0); + + // get the hash from the TxHashRole (QVariant / QString) + uint256 hash; + QString hashQStr = selection.at(0).data(TransactionTableModel::TxHashRole).toString(); + hash.SetHex(hashQStr.toStdString()); + + // Bump tx fee over the walletModel + if (model->bumpFee(hash)) { + // Update the table + model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, true); + } +} + void TransactionView::copyAddress() { GUIUtil::copyEntryData(transactionView, 0, TransactionTableModel::AddressRole); diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 595701cdd9..52e57cae4c 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -76,6 +76,7 @@ private: QDateTimeEdit *dateFrom; QDateTimeEdit *dateTo; QAction *abandonAction; + QAction *bumpFeeAction; QWidget *createDateRangeWidget(); @@ -99,6 +100,7 @@ private Q_SLOTS: void openThirdPartyTxUrl(QString url); void updateWatchOnlyColumn(bool fHaveWatchOnly); void abandonTx(); + void bumpFee(); Q_SIGNALS: void doubleClicked(const QModelIndex&); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index a2a9271904..8df0e481bd 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -8,8 +8,10 @@ #include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" +#include "optionsmodel.h" #include "paymentserver.h" #include "recentrequeststablemodel.h" +#include "sendcoinsdialog.h" #include "transactiontablemodel.h" #include "base58.h" @@ -17,15 +19,18 @@ #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman +#include "policy/rbf.h" #include "sync.h" #include "ui_interface.h" #include "util.h" // for GetBoolArg +#include "wallet/feebumper.h" #include "wallet/wallet.h" #include "wallet/walletdb.h" // for BackupWallet #include <stdint.h> #include <QDebug> +#include <QMessageBox> #include <QSet> #include <QTimer> @@ -63,14 +68,7 @@ CAmount WalletModel::getBalance(const CCoinControl *coinControl) const { if (coinControl) { - CAmount nBalance = 0; - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins, true, coinControl); - BOOST_FOREACH(const COutput& out, vCoins) - if(out.fSpendable) - nBalance += out.tx->tx->vout[out.i].nValue; - - return nBalance; + return wallet->GetAvailableBalance(coinControl); } return wallet->GetBalance(); @@ -595,38 +593,11 @@ bool WalletModel::isSpent(const COutPoint& outpoint) const // AvailableCoins + LockedCoins grouped by wallet address (put change in one group with wallet address) void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) const { - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins); - - LOCK2(cs_main, wallet->cs_wallet); // ListLockedCoins, mapWallet - std::vector<COutPoint> vLockedCoins; - wallet->ListLockedCoins(vLockedCoins); - - // add locked coins - BOOST_FOREACH(const COutPoint& outpoint, vLockedCoins) - { - if (!wallet->mapWallet.count(outpoint.hash)) continue; - int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); - if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true /* spendable */, true /* solvable */, true /* safe */); - if (outpoint.n < out.tx->tx->vout.size() && wallet->IsMine(out.tx->tx->vout[outpoint.n]) == ISMINE_SPENDABLE) - vCoins.push_back(out); - } - - BOOST_FOREACH(const COutput& out, vCoins) - { - COutput cout = out; - - while (wallet->IsChange(cout.tx->tx->vout[cout.i]) && cout.tx->tx->vin.size() > 0 && wallet->IsMine(cout.tx->tx->vin[0])) - { - if (!wallet->mapWallet.count(cout.tx->tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->tx->vin[0].prevout.hash], cout.tx->tx->vin[0].prevout.n, 0 /* depth */, true /* spendable */, true /* solvable */, true /* safe */); + for (auto& group : wallet->ListCoins()) { + auto& resultGroup = mapCoins[QString::fromStdString(CBitcoinAddress(group.first).ToString())]; + for (auto& coin : group.second) { + resultGroup.emplace_back(std::move(coin)); } - - CTxDestination address; - if(!out.fSpendable || !ExtractDestination(cout.tx->tx->vout[cout.i].scriptPubKey, address)) - continue; - mapCoins[QString::fromStdString(CBitcoinAddress(address).ToString())].push_back(out); } } @@ -656,11 +627,7 @@ void WalletModel::listLockedCoins(std::vector<COutPoint>& vOutpts) void WalletModel::loadReceiveRequests(std::vector<std::string>& vReceiveRequests) { - LOCK(wallet->cs_wallet); - BOOST_FOREACH(const PAIRTYPE(CTxDestination, CAddressBookData)& item, wallet->mapAddressBook) - BOOST_FOREACH(const PAIRTYPE(std::string, std::string)& item2, item.second.destdata) - if (item2.first.size() > 2 && item2.first.substr(0,2) == "rr") // receive request - vReceiveRequests.push_back(item2.second); + vReceiveRequests = wallet->GetDestValues("rr"); // receive request } bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t nId, const std::string &sRequest) @@ -680,11 +647,7 @@ bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t bool WalletModel::transactionCanBeAbandoned(uint256 hash) const { - LOCK2(cs_main, wallet->cs_wallet); - const CWalletTx *wtx = wallet->GetWalletTx(hash); - if (!wtx || wtx->isAbandoned() || wtx->GetDepthInMainChain() > 0 || wtx->InMempool()) - return false; - return true; + return wallet->TransactionCanBeAbandoned(hash); } bool WalletModel::abandonTransaction(uint256 hash) const @@ -693,6 +656,84 @@ bool WalletModel::abandonTransaction(uint256 hash) const return wallet->AbandonTransaction(hash); } +bool WalletModel::transactionCanBeBumped(uint256 hash) const +{ + LOCK2(cs_main, wallet->cs_wallet); + const CWalletTx *wtx = wallet->GetWalletTx(hash); + return wtx && SignalsOptInRBF(*wtx) && !wtx->mapValue.count("replaced_by_txid"); +} + +bool WalletModel::bumpFee(uint256 hash) +{ + std::unique_ptr<CFeeBumper> feeBump; + { + LOCK2(cs_main, wallet->cs_wallet); + feeBump.reset(new CFeeBumper(wallet, hash, nTxConfirmTarget, false, 0, true)); + } + if (feeBump->getResult() != BumpFeeResult::OK) + { + QMessageBox::critical(0, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + + (feeBump->getErrors().size() ? QString::fromStdString(feeBump->getErrors()[0]) : "") +")"); + return false; + } + + // allow a user based fee verification + QString questionString = tr("Do you want to increase the fee?"); + questionString.append("<br />"); + CAmount oldFee = feeBump->getOldFee(); + CAmount newFee = feeBump->getNewFee(); + questionString.append("<table style=\"text-align: left;\">"); + questionString.append("<tr><td>"); + questionString.append(tr("Current fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("Increase:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee - oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("New fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee)); + questionString.append("</td></tr></table>"); + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); + confirmationDialog.exec(); + QMessageBox::StandardButton retval = (QMessageBox::StandardButton)confirmationDialog.result(); + + // cancel sign&broadcast if users doesn't want to bump the fee + if (retval != QMessageBox::Yes) { + return false; + } + + WalletModel::UnlockContext ctx(requestUnlock()); + if(!ctx.isValid()) + { + return false; + } + + // sign bumped transaction + bool res = false; + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->signTransaction(wallet); + } + if (!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Can't sign transaction.")); + return false; + } + // commit the bumped transaction + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->commit(wallet); + } + if(!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + + QString::fromStdString(feeBump->getErrors()[0])+")"); + return false; + } + return true; +} + bool WalletModel::isWalletEnabled() { return !GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 78e45dc369..16b0caed4e 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -207,6 +207,9 @@ public: bool transactionCanBeAbandoned(uint256 hash) const; bool abandonTransaction(uint256 hash) const; + bool transactionCanBeBumped(uint256 hash) const; + bool bumpFee(uint256 hash); + static bool isWalletEnabled(); bool hdEnabled() const; diff --git a/src/random.cpp b/src/random.cpp index 6187f16290..de7553c825 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -16,6 +16,8 @@ #include <stdlib.h> #include <limits> +#include <chrono> +#include <thread> #ifndef WIN32 #include <sys/time.h> @@ -32,6 +34,8 @@ #include <sys/sysctl.h> #endif +#include <mutex> + #include <openssl/err.h> #include <openssl/rand.h> @@ -43,15 +47,22 @@ static void RandFailure() static inline int64_t GetPerformanceCounter() { - int64_t nCounter = 0; -#ifdef WIN32 - QueryPerformanceCounter((LARGE_INTEGER*)&nCounter); + // Read the hardware time stamp counter when available. + // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + return __rdtsc(); +#elif !defined(_MSC_VER) && defined(__i386__) + uint64_t r = 0; + __asm__ volatile ("rdtsc" : "=A"(r)); // Constrain the r variable to the eax:edx pair. + return r; +#elif !defined(_MSC_VER) && (defined(__x86_64__) || defined(__amd64__)) + uint64_t r1 = 0, r2 = 0; + __asm__ volatile ("rdtsc" : "=a"(r1), "=d"(r2)); // Constrain r1 to rax and r2 to rdx. + return (r2 << 32) | r1; #else - timeval t; - gettimeofday(&t, NULL); - nCounter = (int64_t)(t.tv_sec * 1000000 + t.tv_usec); + // Fall back to using C++11 clock (usually microsecond or nanosecond precision) + return std::chrono::high_resolution_clock::now().time_since_epoch().count(); #endif - return nCounter; } void RandAddSeed() @@ -192,6 +203,43 @@ void GetRandBytes(unsigned char* buf, int num) } } +static void AddDataToRng(void* data, size_t len); + +void RandAddSeedSleep() +{ + int64_t nPerfCounter1 = GetPerformanceCounter(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + int64_t nPerfCounter2 = GetPerformanceCounter(); + + // Combine with and update state + AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1)); + AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2)); + + memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); + memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); +} + + +static std::mutex cs_rng_state; +static unsigned char rng_state[32] = {0}; +static uint64_t rng_counter = 0; + +static void AddDataToRng(void* data, size_t len) { + CSHA512 hasher; + hasher.Write((const unsigned char*)&len, sizeof(len)); + hasher.Write((const unsigned char*)data, len); + unsigned char buf[64]; + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + memory_cleanse(buf, 64); +} + void GetStrongRandBytes(unsigned char* out, int num) { assert(num <= 32); @@ -207,8 +255,17 @@ void GetStrongRandBytes(unsigned char* out, int num) GetOSRand(buf); hasher.Write(buf, 32); + // Combine with and update state + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + // Produce output - hasher.Finalize(buf); memcpy(out, buf, num); memory_cleanse(buf, 64); } @@ -254,6 +311,8 @@ FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false) bool Random_SanityCheck() { + uint64_t start = GetPerformanceCounter(); + /* This does not measure the quality of randomness, but it does test that * OSRandom() overwrites all 32 bytes of the output given a maximum * number of tries. @@ -280,7 +339,18 @@ bool Random_SanityCheck() tries += 1; } while (num_overwritten < NUM_OS_RANDOM_BYTES && tries < MAX_TRIES); - return (num_overwritten == NUM_OS_RANDOM_BYTES); /* If this failed, bailed out after too many tries */ + if (num_overwritten != NUM_OS_RANDOM_BYTES) return false; /* If this failed, bailed out after too many tries */ + + // Check that GetPerformanceCounter increases at least during a GetOSRand() call + 1ms sleep. + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + uint64_t stop = GetPerformanceCounter(); + if (stop == start) return false; + + // We called GetPerformanceCounter. Use it as entropy. + RAND_add((const unsigned char*)&start, sizeof(start), 1); + RAND_add((const unsigned char*)&stop, sizeof(stop), 1); + + return true; } FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) diff --git a/src/random.h b/src/random.h index 9551e1c461..6a63d57429 100644 --- a/src/random.h +++ b/src/random.h @@ -24,6 +24,13 @@ int GetRandInt(int nMax); uint256 GetRandHash(); /** + * Add a little bit of randomness to the output of GetStrongRangBytes. + * This sleeps for a millisecond, so should only be called when there is + * no other work to be done. + */ +void RandAddSeedSleep(); + +/** * Function to gather random data from multiple sources, failing whenever any * of those source fail to provide a result. */ diff --git a/src/rest.cpp b/src/rest.cpp index 7537ed4502..b08d7153b1 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -42,16 +42,19 @@ static const struct { }; struct CCoin { - uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE uint32_t nHeight; CTxOut out; ADD_SERIALIZE_METHODS; + CCoin() : nHeight(0) {} + CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} + template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTxVer); + uint32_t nTxVerDummy = 0; + READWRITE(nTxVerDummy); READWRITE(nHeight); READWRITE(out); } @@ -509,22 +512,11 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool for (size_t i = 0; i < vOutPoints.size(); i++) { - CCoins coins; - uint256 hash = vOutPoints[i].hash; bool hit = false; - if (view.GetCoins(hash, coins)) { - mempool.pruneSpent(hash, coins); - if (coins.IsAvailable(vOutPoints[i].n)) { - hit = true; - // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if - // n is valid but points to an already spent output (IsNull). - CCoin coin; - coin.nTxVer = coins.nVersion; - coin.nHeight = coins.nHeight; - coin.out = coins.vout.at(vOutPoints[i].n); - assert(!coin.out.IsNull()); - outs.push_back(coin); - } + Coin coin; + if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + hit = true; + outs.emplace_back(std::move(coin)); } hits.push_back(hit); @@ -568,7 +560,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); utxo.push_back(Pair("height", (int32_t)coin.nHeight)); utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e2e244a5f1..96871ce1dc 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -13,6 +13,7 @@ #include "consensus/validation.h" #include "validation.h" #include "core_io.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" @@ -186,7 +187,7 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } - cond_blockchange.notify_all(); + cond_blockchange.notify_all(); } UniValue waitfornewblock(const JSONRPCRequest& request) @@ -412,6 +413,7 @@ UniValue getrawmempool(const JSONRPCRequest& request) throw std::runtime_error( "getrawmempool ( verbose )\n" "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" "\nArguments:\n" "1. verbose (boolean, optional, default=false) True for a json object, false for array of transaction ids\n" "\nResult: (for verbose = false):\n" @@ -686,13 +688,16 @@ UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "getblock \"blockhash\" ( verbose )\n" - "\nIf verbose is false, returns a string that is serialized, hex-encoded data for block 'hash'.\n" - "If verbose is true, returns an Object with information about block <hash>.\n" + "getblock \"blockhash\" ( verbosity ) \n" + "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" + "If verbosity is 1, returns an Object with information about block <hash>.\n" + "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n" "\nArguments:\n" "1. \"blockhash\" (string, required) The block hash\n" - "2. verbose (boolean, optional, default=true) true for a json object, false for the hex encoded data\n" - "\nResult (for verbose = true):\n" + "2. verbosity (numeric, optional, default=1) 0 for hex encoded data, 1 for a json object, and 2 for json object with transaction data\n" + "\nResult (for verbosity = 0):\n" + "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 1):\n" "{\n" " \"hash\" : \"hash\", (string) the block hash (same as provided)\n" " \"confirmations\" : n, (numeric) The number of confirmations, or -1 if the block is not on the main chain\n" @@ -716,8 +721,14 @@ UniValue getblock(const JSONRPCRequest& request) " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\" (string) The hash of the next block\n" "}\n" - "\nResult (for verbose=false):\n" - "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 2):\n" + "{\n" + " ..., Same output as verbosity = 1.\n" + " \"tx\" : [ (array of Objects) The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 \"tx\" result.\n" + " ,...\n" + " ],\n" + " ,... Same output as verbosity = 1.\n" + "}\n" "\nExamples:\n" + HelpExampleCli("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") @@ -728,9 +739,13 @@ UniValue getblock(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); - bool fVerbose = true; - if (request.params.size() > 1) - fVerbose = request.params[1].get_bool(); + int verbosity = 1; + if (request.params.size() > 1) { + if(request.params[1].isNum()) + verbosity = request.params[1].get_int(); + else + verbosity = request.params[1].get_bool() ? 1 : 0; + } if (mapBlockIndex.count(hash) == 0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -749,7 +764,7 @@ UniValue getblock(const JSONRPCRequest& request) // block). throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); - if (!fVerbose) + if (verbosity <= 0) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; @@ -757,7 +772,7 @@ UniValue getblock(const JSONRPCRequest& request) return strHex; } - return blockToJSON(block, pblockindex); + return blockToJSON(block, pblockindex, verbosity >= 2); } struct CCoinsStats @@ -766,13 +781,29 @@ struct CCoinsStats uint256 hashBlock; uint64_t nTransactions; uint64_t nTransactionOutputs; - uint64_t nSerializedSize; uint256 hashSerialized; + uint64_t nDiskSize; CAmount nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nTotalAmount(0) {} }; +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << VARINT(output.second.out.nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { @@ -785,32 +816,29 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - CAmount nTotalAmount = 0; + uint256 prevkey; + std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; i<coins.vout.size(); i++) { - const CTxOut &out = coins.vout[i]; - if (!out.IsNull()) { - stats.nTransactionOutputs++; - ss << VARINT(i+1); - ss << out; - nTotalAmount += out.nValue; - } + COutPoint key; + Coin coin; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - stats.nSerializedSize += 32 + pcursor->GetValueSize(); - ss << VARINT(0); + prevkey = key.hash; + outputs[key.n] = std::move(coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; + stats.nDiskSize = view->EstimateSize(); return true; } @@ -876,8 +904,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"bytes_serialized\": n, (numeric) The serialized size\n" " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -894,8 +922,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); - ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); - ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); + ret.push_back(Pair("hash_serialized_2", stats.hashSerialized.GetHex())); + ret.push_back(Pair("disk_size", stats.nDiskSize)); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -948,37 +976,37 @@ UniValue gettxout(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int n = request.params[1].get_int(); + COutPoint out(hash, n); bool fMempool = true; if (request.params.size() > 2) fMempool = request.params[2].get_bool(); - CCoins coins; + Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins)) + if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { // TODO: filtering spent coins should be done by the CCoinsViewMemPool return NullUniValue; - mempool.pruneSpent(hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } } else { - if (!pcoinsTip->GetCoins(hash, coins)) + if (!pcoinsTip->GetCoin(out, coin)) { return NullUniValue; + } } - if (n<0 || (unsigned int)n>=coins.vout.size() || coins.vout[n].IsNull()) - return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); - if ((unsigned int)coins.nHeight == MEMPOOL_HEIGHT) + if (coin.nHeight == MEMPOOL_HEIGHT) { ret.push_back(Pair("confirmations", 0)); - else - ret.push_back(Pair("confirmations", pindex->nHeight - coins.nHeight + 1)); - ret.push_back(Pair("value", ValueFromAmount(coins.vout[n].nValue))); + } else { + ret.push_back(Pair("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1))); + } + ret.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); ret.push_back(Pair("scriptPubKey", o)); - ret.push_back(Pair("version", coins.nVersion)); - ret.push_back(Pair("coinbase", coins.fCoinBase)); + ret.push_back(Pair("coinbase", (bool)coin.fCoinBase)); return ret; } @@ -1059,6 +1087,17 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse rv.push_back(Pair("startTime", consensusParams.vDeployments[id].nStartTime)); rv.push_back(Pair("timeout", consensusParams.vDeployments[id].nTimeout)); rv.push_back(Pair("since", VersionBitsTipStateSinceHeight(consensusParams, id))); + if (THRESHOLD_STARTED == thresholdState) + { + UniValue statsUV(UniValue::VOBJ); + BIP9Stats statsStruct = VersionBitsTipStatistics(consensusParams, id); + statsUV.push_back(Pair("period", statsStruct.period)); + statsUV.push_back(Pair("threshold", statsStruct.threshold)); + statsUV.push_back(Pair("elapsed", statsStruct.elapsed)); + statsUV.push_back(Pair("count", statsStruct.count)); + statsUV.push_back(Pair("possible", statsStruct.possible)); + rv.push_back(Pair("statistics", statsUV)); + } return rv; } @@ -1104,7 +1143,14 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx (numeric) height of the first block to which the status applies\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " }\n" " }\n" " }\n" "}\n" @@ -1292,7 +1338,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) " \"bytes\": xxxxx, (numeric) Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted\n" " \"usage\": xxxxx, (numeric) Total memory usage for the mempool\n" " \"maxmempool\": xxxxx, (numeric) Maximum memory usage for the mempool\n" - " \"mempoolminfee\": xxxxx (numeric) Minimum fee for tx to be accepted\n" + " \"mempoolminfee\": xxxxx (numeric) Minimum feerate (" + CURRENCY_UNIT + " per KB) for tx to be accepted\n" "}\n" "\nExamples:\n" + HelpExampleCli("getmempoolinfo", "") @@ -1415,13 +1461,79 @@ UniValue reconsiderblock(const JSONRPCRequest& request) return NullUniValue; } +UniValue getchaintxstats(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) + throw std::runtime_error( + "getchaintxstats ( nblocks blockhash )\n" + "\nCompute statistics about the total number and rate of transactions in the chain.\n" + "\nArguments:\n" + "1. nblocks (numeric, optional) Size of the window in number of blocks (default: one month).\n" + "2. \"blockhash\" (string, optional) The hash of the block that ends the window.\n" + "\nResult:\n" + "{\n" + " \"time\": xxxxx, (numeric) The timestamp for the statistics in UNIX format.\n" + " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" + " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getchaintxstats", "") + + HelpExampleRpc("getchaintxstats", "2016") + ); + + const CBlockIndex* pindex; + int blockcount = 30 * 24 * 60 * 60 / Params().GetConsensus().nPowTargetSpacing; // By default: 1 month + + if (request.params.size() > 0 && !request.params[0].isNull()) { + blockcount = request.params[0].get_int(); + } + + bool havehash = request.params.size() > 1 && !request.params[1].isNull(); + uint256 hash; + if (havehash) { + hash = uint256S(request.params[1].get_str()); + } + + { + LOCK(cs_main); + if (havehash) { + auto it = mapBlockIndex.find(hash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + pindex = it->second; + if (!chainActive.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); + } + } else { + pindex = chainActive.Tip(); + } + } + + if (blockcount < 1 || blockcount >= pindex->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 1 and the block's height"); + } + + const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + + UniValue ret(UniValue::VOBJ); + ret.push_back(Pair("time", (int64_t)pindex->nTime)); + ret.push_back(Pair("txcount", (int64_t)pindex->nChainTx)); + ret.push_back(Pair("txrate", ((double)nTxDiff) / nTimeDiff)); + + return ret; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} }, + { "blockchain", "getchaintxstats", &getchaintxstats, true, {"nblocks", "blockhash"} }, { "blockchain", "getbestblockhash", &getbestblockhash, true, {} }, { "blockchain", "getblockcount", &getblockcount, true, {} }, - { "blockchain", "getblock", &getblock, true, {"blockhash","verbose"} }, + { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity|verbose"} }, { "blockchain", "getblockhash", &getblockhash, true, {"height"} }, { "blockchain", "getblockheader", &getblockheader, true, {"blockhash","verbose"} }, { "blockchain", "getchaintips", &getchaintips, true, {} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 941bdd9379..a3a692c14d 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -77,8 +77,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 0, "minconf" }, { "listunspent", 1, "maxconf" }, { "listunspent", 2, "addresses" }, - { "getblock", 1, "verbose" }, + { "listunspent", 4, "query_options" }, + { "getblock", 1, "verbosity" }, { "getblockheader", 1, "verbose" }, + { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "getrawtransaction", 1, "verbose" }, { "createrawtransaction", 0, "inputs" }, @@ -106,6 +108,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getrawmempool", 0, "verbose" }, { "estimatefee", 0, "nblocks" }, { "estimatesmartfee", 0, "nblocks" }, + { "estimatesmartfee", 1, "conservative" }, + { "estimaterawfee", 0, "nblocks" }, + { "estimaterawfee", 1, "threshold" }, + { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "fee_delta" }, { "setban", 2, "bantime" }, { "setban", 3, "absolute" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 4ce52a6c7f..d744269df1 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -797,6 +797,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "estimatefee nblocks\n" + "\nDEPRECATED. Please use estimatesmartfee for more intelligent estimates." "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks. Uses virtual transaction size of transaction\n" "as defined in BIP 141 (witness data is discounted).\n" @@ -828,16 +829,19 @@ UniValue estimatefee(const JSONRPCRequest& request) UniValue estimatesmartfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "estimatesmartfee nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "estimatesmartfee nblocks (conservative)\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid. Uses virtual transaction size as defined\n" "in BIP 141 (witness data is discounted).\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric)\n" + "2. conservative (bool, optional, default=true) Whether to return a more conservative estimate which\n" + " also satisfies a longer history. A conservative estimate potentially returns a higher\n" + " feerate and is more likely to be sufficient for the desired target, but is not as\n" + " responsive to short term drops in the prevailing fee market\n" "\nResult:\n" "{\n" " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" @@ -854,15 +858,102 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)); int nBlocks = request.params[0].get_int(); + bool conservative = true; + if (request.params.size() > 1 && !request.params[1].isNull()) { + RPCTypeCheckArgument(request.params[1], UniValue::VBOOL); + conservative = request.params[1].get_bool(); + } UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool, conservative); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; } +UniValue estimaterawfee(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + throw std::runtime_error( + "estimaterawfee nblocks (threshold horizon)\n" + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + " implementation of fee estimation. The parameters it can be called with\n" + " and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" + " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" + " lower buckets. Default: 0.95\n" + "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" + " Default: 1\n" + "\nResult:\n" + "{\n" + " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " }\n" + " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" + "}\n" + "\n" + "A negative feerate is returned if no answer can be given.\n" + "\nExample:\n" + + HelpExampleCli("estimaterawfee", "6 0.9 1") + ); + + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VNUM)(UniValue::VNUM), true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + int nBlocks = request.params[0].get_int(); + double threshold = 0.95; + if (!request.params[1].isNull()) + threshold = request.params[1].get_real(); + FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; + if (!request.params[2].isNull()) { + int horizonInt = request.params[2].get_int(); + if (horizonInt < 0 || horizonInt > 2) { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); + } else { + horizon = (FeeEstimateHorizon)horizonInt; + } + } + UniValue result(UniValue::VOBJ); + CFeeRate feeRate; + EstimationResult buckets; + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + + result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); + result.push_back(Pair("decay", buckets.decay)); + result.push_back(Pair("scale", (int)buckets.scale)); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("pass", passbucket)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("fail", failbucket)); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- @@ -876,7 +967,9 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, true, {"nblocks","address","maxtries"} }, { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, - { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} }, + { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, + + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 3947fb3f7d..e27c2a77c7 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -219,9 +219,10 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins(oneTxid, coins) && coins.nHeight > 0 && coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsSpent() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + pblockindex = chainActive[coin.nHeight]; + } } if (pblockindex == NULL) @@ -523,6 +524,11 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: UniValue entry(UniValue::VOBJ); entry.push_back(Pair("txid", txin.prevout.hash.ToString())); entry.push_back(Pair("vout", (uint64_t)txin.prevout.n)); + UniValue witness(UniValue::VARR); + for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { + witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + } + entry.push_back(Pair("witness", witness)); entry.push_back(Pair("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); entry.push_back(Pair("sequence", (uint64_t)txin.nSequence)); entry.push_back(Pair("error", strMessage)); @@ -632,9 +638,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view BOOST_FOREACH(const CTxIn& txin, mergedTx.vin) { - const uint256& prevHash = txin.prevout.hash; - CCoins coins; - view.AccessCoins(prevHash); // this is certainly allowed to fail + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. } view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long @@ -686,24 +690,26 @@ UniValue signrawtransaction(const JSONRPCRequest& request) if (nOut < 0) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and not using the local wallet (private keys @@ -761,13 +767,13 @@ UniValue signrawtransaction(const JSONRPCRequest& request) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (coins == NULL || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -839,9 +845,12 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) nMaxRawTxFee = 0; CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsSpent(); + } bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 5c493428d8..c5fbff0077 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -22,6 +22,8 @@ #include <boost/signals2/signal.hpp> #include <boost/thread.hpp> #include <boost/algorithm/string/case_conv.hpp> // for to_upper() +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> #include <memory> // for unique_ptr #include <unordered_map> @@ -42,17 +44,17 @@ static struct CRPCSignals boost::signals2::signal<void (const CRPCCommand&)> PreCommand; } g_rpcSignals; -void RPCServer::OnStarted(boost::function<void ()> slot) +void RPCServer::OnStarted(std::function<void ()> slot) { g_rpcSignals.Started.connect(slot); } -void RPCServer::OnStopped(boost::function<void ()> slot) +void RPCServer::OnStopped(std::function<void ()> slot) { g_rpcSignals.Stopped.connect(slot); } -void RPCServer::OnPreCommand(boost::function<void (const CRPCCommand&)> slot) +void RPCServer::OnPreCommand(std::function<void (const CRPCCommand&)> slot) { g_rpcSignals.PreCommand.connect(boost::bind(slot, _1)); } @@ -432,8 +434,16 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c } // Process expected parameters. int hole = 0; - for (const std::string &argName: argNames) { - auto fr = argsIn.find(argName); + for (const std::string &argNamePattern: argNames) { + std::vector<std::string> vargNames; + boost::algorithm::split(vargNames, argNamePattern, boost::algorithm::is_any_of("|")); + auto fr = argsIn.end(); + for (const std::string & argName : vargNames) { + fr = argsIn.find(argName); + if (fr != argsIn.end()) { + break; + } + } if (fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, @@ -526,7 +536,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface) timerInterface = NULL; } -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds) +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds) { if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); diff --git a/src/rpc/server.h b/src/rpc/server.h index 34a9d3c24c..1e984cbc0d 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -15,8 +15,6 @@ #include <stdint.h> #include <string> -#include <boost/function.hpp> - #include <univalue.h> static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; @@ -25,9 +23,9 @@ class CRPCCommand; namespace RPCServer { - void OnStarted(boost::function<void ()> slot); - void OnStopped(boost::function<void ()> slot); - void OnPreCommand(boost::function<void (const CRPCCommand&)> slot); + void OnStarted(std::function<void ()> slot); + void OnStopped(std::function<void ()> slot); + void OnPreCommand(std::function<void (const CRPCCommand&)> slot); } class CBlockIndex; @@ -115,7 +113,7 @@ public: * This is needed to cope with the case in which there is no HTTP server, but * only GUI RPC console, and to break the dependency of pcserver on httprpc. */ - virtual RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) = 0; + virtual RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) = 0; }; /** Set the factory function for timers */ @@ -129,7 +127,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); * Run func nSeconds from now. * Overrides previous timer <name> (if any). */ -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds); +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds); typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 0c1cfa2718..923ba2c231 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,6 +4,7 @@ #include "scheduler.h" +#include "random.h" #include "reverselock.h" #include <assert.h> @@ -39,6 +40,11 @@ void CScheduler::serviceQueue() // is called. while (!shouldStop()) { try { + if (!shouldStop() && taskQueue.empty()) { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + // Use this chance to get a tiny bit more entropy + RandAddSeedSleep(); + } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. newTaskScheduled.wait(lock); diff --git a/src/scheduler.h b/src/scheduler.h index 613fc1653f..27412a15b4 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -7,8 +7,8 @@ // // NOTE: -// boost::thread / boost::function / boost::chrono should be ported to -// std::thread / std::function / std::chrono when we support C++11. +// boost::thread / boost::chrono should be ported to std::thread / std::chrono +// when we support C++11. // #include <boost/chrono/chrono.hpp> #include <boost/thread.hpp> diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index f4e5313a78..222cff59ea 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1028,7 +1028,7 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& } // Size limits - if (stack.size() + altstack.size() > 1000) + if (stack.size() + altstack.size() > MAX_STACK_SIZE) return set_error(serror, SCRIPT_ERR_STACK_SIZE); } } @@ -1142,24 +1142,24 @@ public: uint256 GetPrevoutHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].prevout; + for (const auto& txin : txTo.vin) { + ss << txin.prevout; } return ss.GetHash(); } uint256 GetSequenceHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].nSequence; + for (const auto& txin : txTo.vin) { + ss << txin.nSequence; } return ss.GetHash(); } uint256 GetOutputsHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vout.size(); n++) { - ss << txTo.vout[n]; + for (const auto& txout : txTo.vout) { + ss << txout; } return ss.GetHash(); } diff --git a/src/script/script.h b/src/script/script.h index d7aaa04f80..95a5999a13 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -30,6 +30,9 @@ static const int MAX_PUBKEYS_PER_MULTISIG = 20; // Maximum script length in bytes static const int MAX_SCRIPT_SIZE = 10000; +// Maximum number of values on script interpreter stack +static const int MAX_STACK_SIZE = 1000; + // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. static const unsigned int LOCKTIME_THRESHOLD = 500000000; // Tue Nov 5 00:53:20 1985 UTC diff --git a/src/support/cleanse.h b/src/support/cleanse.h index 3e02aa8fd1..f020216c73 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -8,6 +8,7 @@ #include <stdlib.h> +// Attempt to overwrite data in the specified memory span. void memory_cleanse(void *ptr, size_t len); #endif // BITCOIN_SUPPORT_CLEANSE_H diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 39fa381dd0..dc5372a070 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -84,36 +84,47 @@ BOOST_AUTO_TEST_CASE(addrman_simple) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 1: Does Addrman respond correctly when empty. - BOOST_CHECK(addrman.size() == 0); + // Test: Does Addrman respond correctly when empty. + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null = addrman.Select(); - BOOST_CHECK(addr_null.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); - // Test 2: Does Addrman::Add work as expected. + // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret1 = addrman.Select(); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 3: Does IP address deduplication work correctly. + // Test: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1_dup, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(!addrman.Add(CAddress(addr1_dup, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); + + // Test: New table has one addr and we add a diff addr we should + // have at least one addr. + // Note that addrman's size cannot be tested reliably after insertion, as + // hash collisions may occur. But we can always be sure of at least one + // success. - // Test 5: New table has one addr and we add a diff addr we should - // have two addrs. CService addr2 = ResolveService("250.1.1.2", 8333); - addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 2); + BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); + BOOST_CHECK(addrman.size() >= 1); - // Test 6: AddrMan::Clear() should empty the new table. + // Test: AddrMan::Clear() should empty the new table. addrman.Clear(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null2 = addrman.Select(); - BOOST_CHECK(addr_null2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null2.ToString(), "[::]:0"); + + // Test: AddrMan::Add multiple addresses works as expected + std::vector<CAddress> vAddr; + vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); + vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); + BOOST_CHECK(addrman.Add(vAddr, source)); + BOOST_CHECK(addrman.size() >= 1); } BOOST_AUTO_TEST_CASE(addrman_ports) @@ -125,26 +136,26 @@ BOOST_AUTO_TEST_CASE(addrman_ports) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CService addr1_port = ResolveService("250.1.1.1", 8334); addrman.Add(CAddress(addr1_port, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(); - BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); - // Test 8: Add same IP but diff port to tried table, it doesn't get added. + // Test: Add same IP but diff port to tried table, it doesn't get added. // Perhaps this is not ideal behavior but it is the current behavior. addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret3 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } @@ -157,25 +168,25 @@ BOOST_AUTO_TEST_CASE(addrman_select) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 9: Select from new with 1 addr in new. + // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret1 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 10: move addr to tried, select from new expected nothing returned. + // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); CAddrInfo addr_ret3 = addrman.Select(); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); // Add three addresses to new table. @@ -199,10 +210,10 @@ BOOST_AUTO_TEST_CASE(addrman_select) addrman.Add(CAddress(addr7, NODE_NONE), ResolveService("250.1.1.3", 8333)); addrman.Good(CAddress(addr7, NODE_NONE)); - // Test 11: 6 addrs + 1 addr from last test = 7. - BOOST_CHECK(addrman.size() == 7); + // Test: 6 addrs + 1 addr from last test = 7. + BOOST_CHECK_EQUAL(addrman.size(), 7); - // Test 12: Select pulls from new and tried regardless of port number. + // Test: Select pulls from new and tried regardless of port number. std::set<uint16_t> ports; for (int i = 0; i < 20; ++i) { ports.insert(addrman.Select().GetPort()); @@ -219,24 +230,24 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 18; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); - //Test 13: No collision in new table yet. - BOOST_CHECK(addrman.size() == i); + //Test: No collision in new table yet. + BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 14: new table collision! + //Test: new table collision! CService addr1 = ResolveService("250.1.1.18"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 17); + BOOST_CHECK_EQUAL(addrman.size(), 17); CService addr2 = ResolveService("250.1.1.19"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 18); + BOOST_CHECK_EQUAL(addrman.size(), 18); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) @@ -248,25 +259,25 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 80; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); addrman.Good(CAddress(addr, NODE_NONE)); - //Test 15: No collision in tried table yet. + //Test: No collision in tried table yet. BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 16: tried table collision! + //Test: tried table collision! CService addr1 = ResolveService("250.1.1.80"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 79); + BOOST_CHECK_EQUAL(addrman.size(), 79); CService addr2 = ResolveService("250.1.1.81"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 80); + BOOST_CHECK_EQUAL(addrman.size(), 80); } BOOST_AUTO_TEST_CASE(addrman_find) @@ -276,7 +287,7 @@ BOOST_AUTO_TEST_CASE(addrman_find) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -289,23 +300,20 @@ BOOST_AUTO_TEST_CASE(addrman_find) addrman.Add(addr2, source2); addrman.Add(addr3, source1); - // Test 17: ensure Find returns an IP matching what we searched on. + // Test: ensure Find returns an IP matching what we searched on. CAddrInfo* info1 = addrman.Find(addr1); - BOOST_CHECK(info1); - if (info1) - BOOST_CHECK(info1->ToString() == "250.1.2.1:8333"); + BOOST_REQUIRE(info1); + BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); // Test 18; Find does not discriminate by port number. CAddrInfo* info2 = addrman.Find(addr2); - BOOST_CHECK(info2); - if (info2 && info1) - BOOST_CHECK(info2->ToString() == info1->ToString()); + BOOST_REQUIRE(info2); + BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); - // Test 19: Find returns another IP matching what we searched on. + // Test: Find returns another IP matching what we searched on. CAddrInfo* info3 = addrman.Find(addr3); - BOOST_CHECK(info3); - if (info3) - BOOST_CHECK(info3->ToString() == "251.255.2.1:8333"); + BOOST_REQUIRE(info3); + BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) @@ -315,7 +323,7 @@ BOOST_AUTO_TEST_CASE(addrman_create) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -323,11 +331,11 @@ BOOST_AUTO_TEST_CASE(addrman_create) int nId; CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); - // Test 20: The result should be the same as the input addr. - BOOST_CHECK(pinfo->ToString() == "250.1.2.1:8333"); + // Test: The result should be the same as the input addr. + BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); CAddrInfo* info2 = addrman.Find(addr1); - BOOST_CHECK(info2->ToString() == "250.1.2.1:8333"); + BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } @@ -338,7 +346,7 @@ BOOST_AUTO_TEST_CASE(addrman_delete) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -346,10 +354,10 @@ BOOST_AUTO_TEST_CASE(addrman_delete) int nId; addrman.Create(addr1, source1, &nId); - // Test 21: Delete should actually delete the addr. - BOOST_CHECK(addrman.size() == 1); + // Test: Delete should actually delete the addr. + BOOST_CHECK_EQUAL(addrman.size(), 1); addrman.Delete(nId); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == NULL); } @@ -361,11 +369,11 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - // Test 22: Sanity check, GetAddr should never return anything if addrman + // Test: Sanity check, GetAddr should never return anything if addrman // is empty. - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); std::vector<CAddress> vAddr1 = addrman.GetAddr(); - BOOST_CHECK(vAddr1.size() == 0); + BOOST_CHECK_EQUAL(vAddr1.size(), 0); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false @@ -380,7 +388,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CNetAddr source1 = ResolveIP("250.1.2.1"); CNetAddr source2 = ResolveIP("250.2.3.3"); - // Test 23: Ensure GetAddr works with new addresses. + // Test: Ensure GetAddr works with new addresses. addrman.Add(addr1, source1); addrman.Add(addr2, source2); addrman.Add(addr3, source1); @@ -388,21 +396,20 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) addrman.Add(addr5, source1); // GetAddr returns 23% of addresses, 23% of 5 is 1 rounded down. - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 24: Ensure GetAddr works with new and tried addresses. + // Test: Ensure GetAddr works with new and tried addresses. addrman.Good(CAddress(addr1, NODE_NONE)); addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 25: Ensure GetAddr still returns 23% when addrman has many addrs. + // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { int octet1 = i % 256; - int octet2 = (i / 256) % 256; - int octet3 = (i / (256 * 2)) % 256; - std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + "." + boost::to_string(octet3) + ".23"; + int octet2 = i >> 8 % 256; + std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + ".1.23"; CAddress addr = CAddress(ResolveService(strAddr), NODE_NONE); - + // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); addrman.Add(addr, ResolveIP(strAddr)); @@ -412,10 +419,10 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) std::vector<CAddress> vAddr = addrman.GetAddr(); size_t percent23 = (addrman.size() * 23) / 100; - BOOST_CHECK(vAddr.size() == percent23); - BOOST_CHECK(vAddr.size() == 461); + BOOST_CHECK_EQUAL(vAddr.size(), percent23); + BOOST_CHECK_EQUAL(vAddr.size(), 461); // (Addrman.size() < number of addresses added) due to address collisions. - BOOST_CHECK(addrman.size() == 2007); + BOOST_CHECK_EQUAL(addrman.size(), 2006); } @@ -438,13 +445,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetTriedBucket(nKey1) == 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); - // Test 26: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); - // Test 27: Two addresses with same IP but different ports can map to + // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); @@ -459,9 +466,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoi.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 28: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // never get more than 8 buckets - BOOST_CHECK(buckets.size() == 8); + BOOST_CHECK_EQUAL(buckets.size(), 8); buckets.clear(); for (int j = 0; j < 255; j++) { @@ -471,9 +478,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoj.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 29: IP addresses in the different groups should map to more than + // Test: IP addresses in the different groups should map to more than // 8 buckets. - BOOST_CHECK(buckets.size() == 160); + BOOST_CHECK_EQUAL(buckets.size(), 160); } BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) @@ -493,16 +500,18 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetNewBucket(nKey1) == 786); + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); - // Test 30: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); - // Test 31: Ports should not effect bucket placement in the addr + // Test: Ports should not effect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetNewBucket(nKey1) == info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); std::set<int> buckets; for (int i = 0; i < 255; i++) { @@ -512,9 +521,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoi.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 32: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // always map to the same bucket. - BOOST_CHECK(buckets.size() == 1); + BOOST_CHECK_EQUAL(buckets.size(), 1); buckets.clear(); for (int j = 0; j < 4 * 255; j++) { @@ -525,7 +534,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 33: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to no more // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -537,7 +546,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 34: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to more // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index c95def5e87..952cf901f0 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "amount.h" +#include "policy/feerate.h" #include "test/test_bitcoin.h" #include <boost/test/unit_test.hpp> diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 31ed1a50b9..4a4e551695 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -17,35 +17,44 @@ #include <boost/test/unit_test.hpp> -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace { +//! equality test +bool operator==(const Coin &a, const Coin &b) { + // Empty Coin objects are always equal. + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && + a.nHeight == b.nHeight && + a.out == b.out; +} + class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map<uint256, CCoins> map_; + std::map<COutPoint, Coin> map_; public: - bool GetCoins(const uint256& txid, CCoins& coins) const + bool GetCoin(const COutPoint& outpoint, Coin& coin) const { - std::map<uint256, CCoins>::const_iterator it = map_.find(txid); + std::map<COutPoint, Coin>::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && insecure_rand() % 2 == 0) { + coin = it->second; + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256& txid) const + bool HaveCoin(const COutPoint& outpoint) const { - CCoins coins; - return GetCoins(txid, coins); + Coin coin; + return GetCoin(outpoint, coin); } uint256 GetBestBlock() const { return hashBestBlock_; } @@ -55,8 +64,8 @@ public: for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -78,9 +87,12 @@ public: { // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); + size_t count = 0; for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); + ++count; } + BOOST_CHECK_EQUAL(GetCacheSize(), count); BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -97,7 +109,7 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // This is a large randomized insert/remove simulation test on a variable-size // stack of caches on top of CCoinsViewTest. // -// It will randomly create/update/delete CCoins entries to a tip of caches, with +// It will randomly create/update/delete Coin entries to a tip of caches, with // txids picked from a limited list of random 256-bit hashes. Occasionally, a // new tip is added to the stack of caches, or the tip is flushed and removed. // @@ -109,13 +121,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) bool removed_all_caches = false; bool reached_4_caches = false; bool added_an_entry = false; + bool added_an_unspendable_entry = false; bool removed_an_entry = false; bool updated_an_entry = false; bool found_an_entry = false; bool missed_an_entry = false; + bool uncached_an_entry = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -133,36 +147,51 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Do a random modification. { uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. - CCoins& coins = result[txid]; - CCoinsModifier entry = stack.back()->ModifyCoins(txid); - BOOST_CHECK(coins == *entry); - if (insecure_rand() % 5 == 0 || coins.IsPruned()) { - if (coins.IsPruned()) { - added_an_entry = true; + Coin& coin = result[COutPoint(txid, 0)]; + const Coin& entry = (insecure_rand() % 500 == 0) ? AccessByTxid(*stack.back(), txid) : stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK(coin == entry); + + if (insecure_rand() % 5 == 0 || coin.IsSpent()) { + Coin newcoin; + newcoin.out.nValue = insecure_rand(); + newcoin.nHeight = 1; + if (insecure_rand() % 16 == 0 && coin.IsSpent()) { + newcoin.out.scriptPubKey.assign(1 + (insecure_rand() & 0x3F), OP_RETURN); + BOOST_CHECK(newcoin.out.scriptPubKey.IsUnspendable()); + added_an_unspendable_entry = true; } else { - updated_an_entry = true; + newcoin.out.scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting + (coin.IsSpent() ? added_an_entry : updated_an_entry) = true; + coin = newcoin; } - coins.nVersion = insecure_rand(); - coins.vout.resize(1); - coins.vout[0].nValue = insecure_rand(); - *entry = coins; + stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), !coin.IsSpent() || insecure_rand() & 1); } else { - coins.Clear(); - entry->Clear(); removed_an_entry = true; + coin.Clear(); + stack.back()->SpendCoin(COutPoint(txid, 0)); } } + // One every 10 iterations, remove a random entry from the cache + if (insecure_rand() % 10) { + COutPoint out(txids[insecure_rand() % txids.size()], 0); + int cacheid = insecure_rand() % stack.size(); + stack[cacheid]->Uncache(out); + uncached_an_entry |= !stack[cacheid]->HaveCoinInCache(out); + } + // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - found_an_entry = true; - } else { - BOOST_CHECK(it->second.IsPruned()); + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); + if (coin.IsSpent()) { missed_an_entry = true; + } else { + BOOST_CHECK(stack.back()->HaveCoinInCache(it->first)); + found_an_entry = true; } } BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { @@ -211,25 +240,27 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(removed_all_caches); BOOST_CHECK(reached_4_caches); BOOST_CHECK(added_an_entry); + BOOST_CHECK(added_an_unspendable_entry); BOOST_CHECK(removed_an_entry); BOOST_CHECK(updated_an_entry); BOOST_CHECK(found_an_entry); BOOST_CHECK(missed_an_entry); + BOOST_CHECK(uncached_an_entry); } -typedef std::tuple<CTransaction,CTxUndo,CCoins> TxData; // Store of all necessary tx and undo data for next test -std::map<uint256, TxData> alltxs; - -TxData &FindRandomFrom(const std::set<uint256> &txidset) { - assert(txidset.size()); - std::set<uint256>::iterator txIt = txidset.lower_bound(GetRandHash()); - if (txIt == txidset.end()) { - txIt = txidset.begin(); +typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; +UtxoData utxoData; + +UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { + assert(utxoSet.size()); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(GetRandHash(), 0)); + if (utxoSetIt == utxoSet.end()) { + utxoSetIt = utxoSet.begin(); } - std::map<uint256, TxData>::iterator txdit = alltxs.find(*txIt); - assert(txdit != alltxs.end()); - return txdit->second; + auto utxoDataIt = utxoData.find(*utxoSetIt); + assert(utxoDataIt != utxoData.end()); + return utxoDataIt; } @@ -242,7 +273,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { bool spent_a_duplicate_coinbase = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -250,10 +281,10 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Track the txids we've used in various sets - std::set<uint256> coinbaseids; - std::set<uint256> disconnectedids; - std::set<uint256> duplicateids; - std::set<uint256> utxoset; + std::set<COutPoint> coinbase_coins; + std::set<COutPoint> disconnected_coins; + std::set<COutPoint> duplicate_coins; + std::set<COutPoint> utxoset; for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { uint32_t randiter = insecure_rand(); @@ -264,23 +295,24 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vin.resize(1); tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate + tx.vout[0].scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting unsigned int height = insecure_rand(); - CCoins oldcoins; + Coin old_coin; // 2/20 times create a new coinbase - if (randiter % 20 < 2 || coinbaseids.size() < 10) { + if (randiter % 20 < 2 || coinbase_coins.size() < 10) { // 1/10 of those times create a duplicate coinbase - if (insecure_rand() % 10 == 0 && coinbaseids.size()) { - TxData &txd = FindRandomFrom(coinbaseids); + if (insecure_rand() % 10 == 0 && coinbase_coins.size()) { + auto utxod = FindRandomFrom(coinbase_coins); // Reuse the exact same coinbase - tx = std::get<0>(txd); + tx = std::get<0>(utxod->second); // shouldn't be available for reconnection if its been duplicated - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); - duplicateids.insert(tx.GetHash()); + duplicate_coins.insert(utxod->first); } else { - coinbaseids.insert(tx.GetHash()); + coinbase_coins.insert(COutPoint(tx.GetHash(), 0)); } assert(CTransaction(tx).IsCoinBase()); } @@ -288,114 +320,118 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // 17/20 times reconnect previous or add a regular tx else { - uint256 prevouthash; + COutPoint prevout; // 1/20 times reconnect a previously disconnected tx - if (randiter % 20 == 2 && disconnectedids.size()) { - TxData &txd = FindRandomFrom(disconnectedids); - tx = std::get<0>(txd); - prevouthash = tx.vin[0].prevout.hash; - if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevouthash)) { - disconnectedids.erase(tx.GetHash()); + if (randiter % 20 == 2 && disconnected_coins.size()) { + auto utxod = FindRandomFrom(disconnected_coins); + tx = std::get<0>(utxod->second); + prevout = tx.vin[0].prevout; + if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevout)) { + disconnected_coins.erase(utxod->first); continue; } // If this tx is already IN the UTXO, then it must be a coinbase, and it must be a duplicate - if (utxoset.count(tx.GetHash())) { + if (utxoset.count(utxod->first)) { assert(CTransaction(tx).IsCoinBase()); - assert(duplicateids.count(tx.GetHash())); + assert(duplicate_coins.count(utxod->first)); } - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); } // 16/20 times create a regular tx else { - TxData &txd = FindRandomFrom(utxoset); - prevouthash = std::get<0>(txd).GetHash(); + auto utxod = FindRandomFrom(utxoset); + prevout = utxod->first; // Construct the tx to spend the coins of prevouthash - tx.vin[0].prevout.hash = prevouthash; - tx.vin[0].prevout.n = 0; + tx.vin[0].prevout = prevout; assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevouthash]; + old_coin = result[prevout]; // Update the expected result of prevouthash to know these coins are spent - result[prevouthash].Clear(); + result[prevout].Clear(); - utxoset.erase(prevouthash); + utxoset.erase(prevout); // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) + if (duplicate_coins.count(prevout)) { spent_a_duplicate_coinbase = true; + } } // Update the expected result to know about the new output coins - result[tx.GetHash()].FromTx(tx, height); + assert(tx.vout.size() == 1); + const COutPoint outpoint(tx.GetHash(), 0); + result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); // Call UpdateCoins on the top cache CTxUndo undo; UpdateCoins(tx, *(stack.back()), undo, height); // Update the utxo set for future spends - utxoset.insert(tx.GetHash()); + utxoset.insert(outpoint); // Track this tx and undo info to use later - alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); - } - - //1/20 times undo a previous transaction - else if (utxoset.size()) { - TxData &txd = FindRandomFrom(utxoset); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,old_coin)); + } else if (utxoset.size()) { + //1/20 times undo a previous transaction + auto utxod = FindRandomFrom(utxoset); - CTransaction &tx = std::get<0>(txd); - CTxUndo &undo = std::get<1>(txd); - CCoins &origcoins = std::get<2>(txd); - - uint256 undohash = tx.GetHash(); + CTransaction &tx = std::get<0>(utxod->second); + CTxUndo &undo = std::get<1>(utxod->second); + Coin &orig_coin = std::get<2>(utxod->second); // Update the expected result // Remove new outputs - result[undohash].Clear(); + result[utxod->first].Clear(); // If not coinbase restore prevout if (!tx.IsCoinBase()) { - result[tx.vin[0].prevout.hash] = origcoins; + result[tx.vin[0].prevout] = orig_coin; } // Disconnect the tx from the current UTXO // See code in DisconnectBlock // remove outputs - { - CCoinsModifier outs = stack.back()->ModifyCoins(undohash); - outs->Clear(); - } + stack.back()->SpendCoin(utxod->first); // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const CTxInUndo &undoin = undo.vprevout[0]; - ApplyTxInUndo(undoin, *(stack.back()), out); + Coin coin = undo.vprevout[0]; + ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection - disconnectedids.insert(undohash); + disconnected_coins.insert(utxod->first); // Update the utxoset - utxoset.erase(undohash); + utxoset.erase(utxod->first); if (!tx.IsCoinBase()) - utxoset.insert(tx.vin[0].prevout.hash); + utxoset.insert(tx.vin[0].prevout); } // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - } else { - BOOST_CHECK(it->second.IsPruned()); - } + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); } } + // One every 10 iterations, remove a random entry from the cache + if (utxoset.size() > 1 && insecure_rand() % 30) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(utxoset)->first); + } + if (disconnected_coins.size() > 1 && insecure_rand() % 30) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(disconnected_coins)->first); + } + if (duplicate_coins.size() > 1 && insecure_rand() % 30) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(duplicate_coins)->first); + } + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && insecure_rand() % 2 == 0) { @@ -433,53 +469,36 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); - CCoins cc1; + CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + Coin cc1; ss1 >> cc1; - BOOST_CHECK_EQUAL(cc1.nVersion, 1); BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); - BOOST_CHECK_EQUAL(cc1.vout.size(), 2); - BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); - BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); - BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); - BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + BOOST_CHECK_EQUAL(cc1.out.nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); - CCoins cc2; + CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + Coin cc2; ss2 >> cc2; - BOOST_CHECK_EQUAL(cc2.nVersion, 1); BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); - BOOST_CHECK_EQUAL(cc2.vout.size(), 17); - for (int i = 0; i < 17; i++) { - BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); - } - BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); - BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + BOOST_CHECK_EQUAL(cc2.out.nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ssx(SER_DISK, CLIENT_VERSION); - BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); - - CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); - CCoins cc3; + CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + Coin cc3; ss3 >> cc3; - BOOST_CHECK_EQUAL(cc3.nVersion, 0); BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); - BOOST_CHECK_EQUAL(cc3.vout.size(), 1); - BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); - BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); - BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + BOOST_CHECK_EQUAL(cc3.out.nValue, 0); + BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); try { - CCoins cc4; + Coin cc4; ss4 >> cc4; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { @@ -490,16 +509,16 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); - CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { - CCoins cc5; + Coin cc5; ss5 >> cc5; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { } } -const static uint256 TXID; +const static COutPoint OUTPOINT; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -514,15 +533,15 @@ const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; const static auto CLEAN_FLAGS = {char(0), FRESH}; const static auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins& coins) +void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsSpent()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + coin.out.nValue = value; + coin.nHeight = 1; + assert(!coin.IsSpent()); } } @@ -535,25 +554,23 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + SetCoinsValue(value, entry.coin); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coin.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); @@ -581,10 +598,10 @@ public: CCoinsViewCacheTest cache{&base}; }; -void CheckAccessCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoins(TXID); + test.cache.AccessCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -603,39 +620,39 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Base Cache Result Cache Result * Value Value Value Flags Flags */ - CheckAccessCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); } -void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - SetCoinsValue(modify_value, *test.cache.ModifyCoins(TXID)); + test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -645,79 +662,55 @@ void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_va BOOST_CHECK_EQUAL(result_flags, expected_flags); }; -BOOST_AUTO_TEST_CASE(ccoins_modify) +BOOST_AUTO_TEST_CASE(ccoins_spend) { - /* Check ModifyCoin behavior, requesting a coin from a cache view layered on - * top of a base view, writing a modification to the coin, and then checking + /* Check SpendCoin behavior, requesting a coin from a cache view layered on + * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. * - * Base Cache Write Result Cache Result - * Value Value Value Value Flags Flags + * Base Cache Result Cache Result + * Value Value Value Flags Flags */ - CheckModifyCoins(ABSENT, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(ABSENT, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(PRUNED, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); + CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } -void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); CAmount result_value; char result_flags; try { - SetCoinsValue(modify_value, *test.cache.ModifyNewCoins(TXID, coinbase)); + CTxOut output; + output.nValue = modify_value; + test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + test.cache.SelfTest(); GetCoinsMapEntry(test.cache.map(), result_value, result_flags); } catch (std::logic_error& e) { result_value = FAIL; @@ -728,64 +721,46 @@ void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount mo BOOST_CHECK_EQUAL(result_flags, expected_flags); } -// Simple wrapper for CheckModifyNewCoinsBase function above that loops through +// Simple wrapper for CheckAddCoinBase function above that loops through // different possible base_values, making sure each one gives the same results. -// This wrapper lets the modify_new test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::ModifyNewCoins implementation +// This wrapper lets the coins_add test below be shorter and less repetitive, +// while still verifying that the CoinsViewCache::AddCoin implementation // ignores base values. template <typename... Args> -void CheckModifyNewCoins(Args&&... args) +void CheckAddCoin(Args&&... args) { for (CAmount base_value : {ABSENT, PRUNED, VALUE1}) - CheckModifyNewCoinsBase(base_value, std::forward<Args>(args)...); + CheckAddCoinBase(base_value, std::forward<Args>(args)...); } -BOOST_AUTO_TEST_CASE(ccoins_modify_new) +BOOST_AUTO_TEST_CASE(ccoins_add) { - /* Check ModifyNewCoin behavior, requesting a new coin from a cache view, + /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the ModifyNewCoin coinbase argument set to false, and to true. + * with the AddCoin potential_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result Coinbase - * Value Value Value Flags Flags + * Cache Write Result Cache Result potential_overwrite + * Value Value Value Flags Flags */ - CheckModifyNewCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY , false); - CheckModifyNewCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, 0 , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); } void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 5c054ed3e8..e35a7ce569 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -349,7 +349,7 @@ ["2147483647", "0x04 0xFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["2147483648", "0x05 0x0000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["549755813887", "0x05 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], -["549755813888", "0x06 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], +["549755813888", "0x06 0x000000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["9223372036854775807", "0x08 0xFFFFFFFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["-1", "0x01 0x81 EQUAL", "P2SH,STRICTENC", "OK", "Numbers are little-endian with the MSB being a sign bit"], ["-127", "0x01 0xFF EQUAL", "P2SH,STRICTENC", "OK"], diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d8de765db1..bb7e473248 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -128,6 +128,23 @@ BOOST_AUTO_TEST_CASE(siphash) tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); + + // Check consistency between CSipHasher and SipHashUint256[Extra]. + FastRandomContext ctx; + for (int i = 0; i < 16; ++i) { + uint64_t k1 = ctx.rand64(); + uint64_t k2 = ctx.rand64(); + uint256 x = GetRandHash(); + uint32_t n = ctx.rand32(); + uint8_t nb[4]; + WriteLE32(nb, n); + CSipHasher sip256(k1, k2); + sip256.Write(x.begin(), 32); + CSipHasher sip288 = sip256; + sip288.Write(nb, 4); + BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); + BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index d52104b4cc..656aec606b 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -39,17 +39,18 @@ static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) BOOST_AUTO_TEST_CASE(block_subsidy_test) { - TestBlockSubsidyHalvings(Params(CBaseChainParams::MAIN).GetConsensus()); // As in main + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + TestBlockSubsidyHalvings(chainParams->GetConsensus()); // As in main TestBlockSubsidyHalvings(150); // As in regtest TestBlockSubsidyHalvings(1000); // Just another interval } BOOST_AUTO_TEST_CASE(subsidy_limit_test) { - const Consensus::Params& consensusParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); CAmount nSum = 0; for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) { - CAmount nSubsidy = GetBlockSubsidy(nHeight, consensusParams); + CAmount nSubsidy = GetBlockSubsidy(nHeight, chainParams->GetConsensus()); BOOST_CHECK(nSubsidy <= 50 * COIN); nSum += nSubsidy * 1000; BOOST_CHECK(MoneyRange(nSum)); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 41f42c7b88..a40060e657 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -6,6 +6,7 @@ #include "coins.h" #include "consensus/consensus.h" #include "consensus/merkle.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "miner.h" @@ -194,7 +195,8 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { // Note that by default, these tests run with size accounting enabled. - const CChainParams& chainparams = Params(CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const CChainParams& chainparams = *chainParams; CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; std::unique_ptr<CBlockTemplate> pblocktemplate; CMutableTransaction tx,tx2; diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index ed6782ea34..6bfd315647 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -50,8 +50,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int blocknum = 0; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs @@ -75,20 +75,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, ++blocknum); block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(feeEst.estimateSmartFee(1, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(3, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(4, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(8, &answerFound, mpool) == feeEst.estimateFee(8) && answerFound == 8); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee); } } @@ -105,13 +99,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - if (i > 1) { + if (i % 2 == 0) { //At scale 2, test logic is only correct for even targets BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); } - else { - BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) { + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -140,10 +135,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); } - int answerFound; for (int i = 1; i < 10;i++) { BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(feeEst.estimateSmartFee(i, &answerFound, mpool).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); } // Mine all those transactions @@ -156,16 +149,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265); + mpool.removeForBlock(block, 266); block.clear(); BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 400 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) { + while (blocknum < 665) { for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; @@ -181,7 +174,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) block.clear(); } BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); - for (int i = 2; i < 10; i++) { + for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } @@ -191,7 +184,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateFee(i).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 4ca6f1caf0..3b79f8000d 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -16,69 +16,59 @@ BOOST_FIXTURE_TEST_SUITE(pow_tests, BasicTestingSetup) /* Test calculation of next difficulty target with no constraints applying */ BOOST_AUTO_TEST_CASE(get_next_work) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1261130161; // Block #30240 CBlockIndex pindexLast; pindexLast.nHeight = 32255; pindexLast.nTime = 1262152739; // Block #32255 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00d86a); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00d86a); } /* Test the constraint on the upper bound for next work */ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1231006505; // Block #0 CBlockIndex pindexLast; pindexLast.nHeight = 2015; pindexLast.nTime = 1233061996; // Block #2015 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00ffff); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00ffff); } /* Test the constraint on the lower bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1279008237; // Block #66528 CBlockIndex pindexLast; pindexLast.nHeight = 68543; pindexLast.nTime = 1279297671; // Block #68543 pindexLast.nBits = 0x1c05a3f4; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1c0168fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1c0168fd); } /* Test the constraint on the upper bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1263163443; // NOTE: Not an actual block time CBlockIndex pindexLast; pindexLast.nHeight = 46367; pindexLast.nTime = 1269211443; // Block #46367 pindexLast.nBits = 0x1c387f6f; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00e1fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00e1fd); } BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); std::vector<CBlockIndex> blocks(10000); for (int i = 0; i < 10000; i++) { blocks[i].pprev = i ? &blocks[i - 1] : NULL; blocks[i].nHeight = i; - blocks[i].nTime = 1269211443 + i * params.nPowTargetSpacing; + blocks[i].nTime = 1269211443 + i * chainParams->GetConsensus().nPowTargetSpacing; blocks[i].nBits = 0x207fffff; /* target 0x7fffff000... */ blocks[i].nChainWork = i ? blocks[i - 1].nChainWork + GetBlockProof(blocks[i - 1]) : arith_uint256(0); } @@ -88,7 +78,7 @@ BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) CBlockIndex *p2 = &blocks[GetRand(10000)]; CBlockIndex *p3 = &blocks[GetRand(10000)]; - int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, params); + int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, chainParams->GetConsensus()); BOOST_CHECK_EQUAL(tdiff, p1->GetBlockTime() - p2->GetBlockTime()); } } diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index f8fd8cc30c..0789b2e80c 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/tx_verify.h" #include "core_io.h" #include "key.h" #include "keystore.h" @@ -111,7 +112,8 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(CCoins(txFrom, 0), txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + const CTxOut& output = txFrom.vout[txTo[i].vin[0].prevout.n]; + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else @@ -315,7 +317,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops)); txFrom.vout[6].nValue = 6000; - coins.ModifyCoins(txFrom.GetHash())->FromTx(txFrom, 0); + AddCoins(coins, txFrom, 0); CMutableTransaction txTo; txTo.vout.resize(1); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 5279cb243b..2f7c22084e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "data/sighash.json.h" #include "hash.h" -#include "validation.h" // For CheckTransaction #include "script/interpreter.h" #include "script/script.h" #include "serialize.h" diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 13d8911f03..4e117448fe 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validation.h" +#include "consensus/tx_verify.h" #include "pubkey.h" #include "key.h" #include "script/script.h" @@ -102,7 +102,7 @@ void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableT spendingTx.vout[0].nValue = 1; spendingTx.vout[0].scriptPubKey = CScript(); - coins.ModifyCoins(creationTx.GetHash())->FromTx(creationTx, 0); + AddCoins(coins, creationTx, 0); } BOOST_AUTO_TEST_CASE(GetTxSigOpCost) diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index c4983f6f5c..de14251601 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -59,9 +59,8 @@ bool read_stdin(std::vector<char> &data) { return length==0; } -int main(int argc, char **argv) +int do_fuzz() { - ECCVerifyHandle globalVerifyHandle; std::vector<char> buffer; if (!read_stdin(buffer)) return 0; @@ -169,8 +168,8 @@ int main(int argc, char **argv) { try { - CCoins block; - ds >> block; + Coin coin; + ds >> coin; } catch (const std::ios_base::failure& e) {return 0;} break; } @@ -256,3 +255,23 @@ int main(int argc, char **argv) return 0; } +int main(int argc, char **argv) +{ + ECCVerifyHandle globalVerifyHandle; +#ifdef __AFL_INIT + // Enable AFL deferred forkserver mode. Requires compilation using + // afl-clang-fast++. See fuzzing.md for details. + __AFL_INIT(); +#endif + +#ifdef __AFL_LOOP + // Enable AFL persistent mode. Requires compilation using afl-clang-fast++. + // See fuzzing.md for details. + while (__AFL_LOOP(1000)) { + do_fuzz(); + } + return 0; +#else + return do_fuzz(); +#endif +} diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp new file mode 100644 index 0000000000..b7affaacde --- /dev/null +++ b/src/test/torcontrol_tests.cpp @@ -0,0 +1,199 @@ +// Copyright (c) 2017 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include "test/test_bitcoin.h" +#include "torcontrol.cpp" + +#include <boost/test/unit_test.hpp> + + +BOOST_FIXTURE_TEST_SUITE(torcontrol_tests, BasicTestingSetup) + +void CheckSplitTorReplyLine(std::string input, std::string command, std::string args) +{ + BOOST_TEST_MESSAGE(std::string("CheckSplitTorReplyLine(") + input + ")"); + auto ret = SplitTorReplyLine(input); + BOOST_CHECK_EQUAL(ret.first, command); + BOOST_CHECK_EQUAL(ret.second, args); +} + +BOOST_AUTO_TEST_CASE(util_SplitTorReplyLine) +{ + // Data we should receive during normal usage + CheckSplitTorReplyLine( + "PROTOCOLINFO PIVERSION", + "PROTOCOLINFO", "PIVERSION"); + CheckSplitTorReplyLine( + "AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", + "AUTH", "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\""); + CheckSplitTorReplyLine( + "AUTH METHODS=NULL", + "AUTH", "METHODS=NULL"); + CheckSplitTorReplyLine( + "AUTH METHODS=HASHEDPASSWORD", + "AUTH", "METHODS=HASHEDPASSWORD"); + CheckSplitTorReplyLine( + "VERSION Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", + "VERSION", "Tor=\"0.2.9.8 (git-a0df013ea241b026)\""); + CheckSplitTorReplyLine( + "AUTHCHALLENGE SERVERHASH=aaaa SERVERNONCE=bbbb", + "AUTHCHALLENGE", "SERVERHASH=aaaa SERVERNONCE=bbbb"); + + // Other valid inputs + CheckSplitTorReplyLine("COMMAND", "COMMAND", ""); + CheckSplitTorReplyLine("COMMAND SOME ARGS", "COMMAND", "SOME ARGS"); + + // These inputs are valid because PROTOCOLINFO accepts an OtherLine that is + // just an OptArguments, which enables multiple spaces to be present + // between the command and arguments. + CheckSplitTorReplyLine("COMMAND ARGS", "COMMAND", " ARGS"); + CheckSplitTorReplyLine("COMMAND EVEN+more ARGS", "COMMAND", " EVEN+more ARGS"); +} + +void CheckParseTorReplyMapping(std::string input, std::map<std::string,std::string> expected) +{ + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(") + input + ")"); + auto ret = ParseTorReplyMapping(input); + BOOST_CHECK_EQUAL(ret.size(), expected.size()); + auto r_it = ret.begin(); + auto e_it = expected.begin(); + while (r_it != ret.end() && e_it != expected.end()) { + BOOST_CHECK_EQUAL(r_it->first, e_it->first); + BOOST_CHECK_EQUAL(r_it->second, e_it->second); + r_it++; + e_it++; + } +} + +BOOST_AUTO_TEST_CASE(util_ParseTorReplyMapping) +{ + // Data we should receive during normal usage + CheckParseTorReplyMapping( + "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", { + {"METHODS", "COOKIE,SAFECOOKIE"}, + {"COOKIEFILE", "/home/x/.tor/control_auth_cookie"}, + }); + CheckParseTorReplyMapping( + "METHODS=NULL", { + {"METHODS", "NULL"}, + }); + CheckParseTorReplyMapping( + "METHODS=HASHEDPASSWORD", { + {"METHODS", "HASHEDPASSWORD"}, + }); + CheckParseTorReplyMapping( + "Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", { + {"Tor", "0.2.9.8 (git-a0df013ea241b026)"}, + }); + CheckParseTorReplyMapping( + "SERVERHASH=aaaa SERVERNONCE=bbbb", { + {"SERVERHASH", "aaaa"}, + {"SERVERNONCE", "bbbb"}, + }); + CheckParseTorReplyMapping( + "ServiceID=exampleonion1234", { + {"ServiceID", "exampleonion1234"}, + }); + CheckParseTorReplyMapping( + "PrivateKey=RSA1024:BLOB", { + {"PrivateKey", "RSA1024:BLOB"}, + }); + CheckParseTorReplyMapping( + "ClientAuth=bob:BLOB", { + {"ClientAuth", "bob:BLOB"}, + }); + + // Other valid inputs + CheckParseTorReplyMapping( + "Foo=Bar=Baz Spam=Eggs", { + {"Foo", "Bar=Baz"}, + {"Spam", "Eggs"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar=Baz\"", { + {"Foo", "Bar=Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar Baz\"", { + {"Foo", "Bar Baz"}, + }); + + // Escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\ Baz\"", { + {"Foo", "Bar Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\Baz\"", { + {"Foo", "BarBaz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\@Baz\"", { + {"Foo", "Bar@Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\"Baz\" Spam=\"\\\"Eggs\\\"\"", { + {"Foo", "Bar\"Baz"}, + {"Spam", "\"Eggs\""}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\\Baz\"", { + {"Foo", "Bar\\Baz"}, + }); + + // C escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\nBaz\\t\" Spam=\"\\rEggs\" Octals=\"\\1a\\11\\17\\18\\81\\377\\378\\400\\2222\" Final=Check", { + {"Foo", "Bar\nBaz\t"}, + {"Spam", "\rEggs"}, + {"Octals", "\1a\11\17\1" "881\377\37" "8\40" "0\222" "2"}, + {"Final", "Check"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Escaped=\"Escape\\\\\"", { + {"Valid", "Mapping"}, + {"Escaped", "Escape\\"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Bare=\"Escape\\\"", {}); + CheckParseTorReplyMapping( + "OneOctal=\"OneEnd\\1\" TwoOctal=\"TwoEnd\\11\"", { + {"OneOctal", "OneEnd\1"}, + {"TwoOctal", "TwoEnd\11"}, + }); + + // Special handling for null case + // (needed because string comparison reads the null as end-of-string) + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(Null=\"\\0\")")); + auto ret = ParseTorReplyMapping("Null=\"\\0\""); + BOOST_CHECK_EQUAL(ret.size(), 1); + auto r_it = ret.begin(); + BOOST_CHECK_EQUAL(r_it->first, "Null"); + BOOST_CHECK_EQUAL(r_it->second.size(), 1); + BOOST_CHECK_EQUAL(r_it->second[0], '\0'); + + // A more complex valid grammar. PROTOCOLINFO accepts a VersionLine that + // takes a key=value pair followed by an OptArguments, making this valid. + // Because an OptArguments contains no semantic data, there is no point in + // parsing it. + CheckParseTorReplyMapping( + "SOME=args,here MORE optional=arguments here", { + {"SOME", "args,here"}, + }); + + // Inputs that are effectively invalid under the target grammar. + // PROTOCOLINFO accepts an OtherLine that is just an OptArguments, which + // would make these inputs valid. However, + // - This parser is never used in that situation, because the + // SplitTorReplyLine parser enables OtherLine to be skipped. + // - Even if these were valid, an OptArguments contains no semantic data, + // so there is no point in parsing it. + CheckParseTorReplyMapping("ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("EVEN more=ARGS", {}); + CheckParseTorReplyMapping("EVEN+more ARGS", {}); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 3b5da4980b..5c7516fbf1 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -8,11 +8,12 @@ #include "clientversion.h" #include "checkqueue.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "core_io.h" #include "key.h" #include "keystore.h" -#include "validation.h" // For CheckTransaction +#include "validation.h" #include "policy/policy.h" #include "script/script.h" #include "script/sign.h" @@ -306,14 +307,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50*CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21*CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22*CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } @@ -469,19 +470,20 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for (int i=0; i<20; i++) threadGroup.create_thread(boost::bind(&CCheckQueue<CScriptCheck>::Thread, boost::ref(scriptcheckqueue))); - CCoins coins; - coins.nVersion = 1; - coins.fCoinBase = false; + std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { - CTxOut txout; - txout.nValue = 1000; - txout.scriptPubKey = scriptPubKey; - coins.vout.push_back(txout); + Coin coin; + coin.nHeight = 1; + coin.fCoinBase = false; + coin.out.nValue = 1000; + coin.out.scriptPubKey = scriptPubKey; + coins.emplace_back(std::move(coin)); } for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector<CScriptCheck> vChecks; - CScriptCheck check(coins, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); + const CTxOut& output = coins[tx.vin[i].prevout.n].out; + CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); control.Add(vChecks); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 5da4907f06..10330c0c23 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -17,8 +17,6 @@ #include <boost/test/unit_test.hpp> -extern std::map<std::string, std::string> mapArgs; - BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_criticalsection) @@ -100,52 +98,67 @@ BOOST_AUTO_TEST_CASE(util_DateTimeStrFormat) BOOST_CHECK_EQUAL(DateTimeStrFormat("%a, %d %b %Y %H:%M:%S +0000", 1317425777), "Fri, 30 Sep 2011 23:36:17 +0000"); } +class TestArgsManager : public ArgsManager +{ +public: + std::map<std::string, std::string>& GetMapArgs() + { + return mapArgs; + }; + const std::map<std::string, std::vector<std::string> >& GetMapMultiArgs() + { + return mapMultiArgs; + }; +}; + BOOST_AUTO_TEST_CASE(util_ParseParameters) { + TestArgsManager testArgs; const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; - ParseParameters(0, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(0, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(1, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(1, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(5, (char**)argv_test); + testArgs.ParseParameters(5, (char**)argv_test); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(mapArgs.size() == 3 && mapMultiArgs.size() == 3); - BOOST_CHECK(IsArgSet("-a") && IsArgSet("-b") && IsArgSet("-ccc") - && !IsArgSet("f") && !IsArgSet("-d")); - BOOST_CHECK(mapMultiArgs.count("-a") && mapMultiArgs.count("-b") && mapMultiArgs.count("-ccc") - && !mapMultiArgs.count("f") && !mapMultiArgs.count("-d")); - - BOOST_CHECK(mapArgs["-a"] == "" && mapArgs["-ccc"] == "multiple"); - BOOST_CHECK(mapMultiArgs.at("-ccc").size() == 2); + BOOST_CHECK(testArgs.GetMapArgs().size() == 3 && testArgs.GetMapMultiArgs().size() == 3); + BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") + && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); + BOOST_CHECK(testArgs.GetMapMultiArgs().count("-a") && testArgs.GetMapMultiArgs().count("-b") && testArgs.GetMapMultiArgs().count("-ccc") + && !testArgs.GetMapMultiArgs().count("f") && !testArgs.GetMapMultiArgs().count("-d")); + + BOOST_CHECK(testArgs.GetMapArgs()["-a"] == "" && testArgs.GetMapArgs()["-ccc"] == "multiple"); + BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } BOOST_AUTO_TEST_CASE(util_GetArg) { - mapArgs.clear(); - mapArgs["strtest1"] = "string..."; + TestArgsManager testArgs; + testArgs.GetMapArgs().clear(); + testArgs.GetMapArgs()["strtest1"] = "string..."; // strtest2 undefined on purpose - mapArgs["inttest1"] = "12345"; - mapArgs["inttest2"] = "81985529216486895"; + testArgs.GetMapArgs()["inttest1"] = "12345"; + testArgs.GetMapArgs()["inttest2"] = "81985529216486895"; // inttest3 undefined on purpose - mapArgs["booltest1"] = ""; + testArgs.GetMapArgs()["booltest1"] = ""; // booltest2 undefined on purpose - mapArgs["booltest3"] = "0"; - mapArgs["booltest4"] = "1"; - - BOOST_CHECK_EQUAL(GetArg("strtest1", "default"), "string..."); - BOOST_CHECK_EQUAL(GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(GetArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(GetArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(GetArg("inttest3", -1), -1); - BOOST_CHECK_EQUAL(GetBoolArg("booltest1", false), true); - BOOST_CHECK_EQUAL(GetBoolArg("booltest2", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest3", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest4", false), true); + testArgs.GetMapArgs()["booltest3"] = "0"; + testArgs.GetMapArgs()["booltest4"] = "1"; + + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest4", false), true); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index e2b5573abd..79405ec4d1 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -209,7 +209,8 @@ BOOST_AUTO_TEST_CASE(versionbits_test) } // Sanity checks of version bit deployments - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); for (int i=0; i<(int) Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { uint32_t bitmask = VersionBitsMask(mainnetParams, (Consensus::DeploymentPos)i); // Make sure that no deployment tries to set an invalid bit. @@ -235,7 +236,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) { // Check that ComputeBlockVersion will set the appropriate bit correctly // on mainnet. - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); // Use the TESTDUMMY deployment for testing purposes. int64_t bit = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index c1bd95b00f..8a37139f1d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +15,6 @@ #include <set> #include <stdlib.h> -#include <boost/function.hpp> #include <boost/bind.hpp> #include <boost/signals2/signal.hpp> #include <boost/foreach.hpp> @@ -73,8 +73,8 @@ public: class TorControlConnection { public: - typedef boost::function<void(TorControlConnection&)> ConnectionCB; - typedef boost::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; + typedef std::function<void(TorControlConnection&)> ConnectionCB; + typedef std::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; /** Create a new TorControlConnection. */ @@ -105,9 +105,9 @@ public: boost::signals2::signal<void(TorControlConnection &,const TorControlReply &)> async_handler; private: /** Callback when ready for use */ - boost::function<void(TorControlConnection&)> connected; + std::function<void(TorControlConnection&)> connected; /** Callback when connection lost */ - boost::function<void(TorControlConnection&)> disconnected; + std::function<void(TorControlConnection&)> disconnected; /** Libevent event base */ struct event_base *base; /** Connection to control socket */ @@ -250,6 +250,8 @@ bool TorControlConnection::Command(const std::string &cmd, const ReplyHandlerCB& /* Split reply line in the form 'AUTH METHODS=...' into a type * 'AUTH' and arguments 'METHODS=...'. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21) and AUTHCHALLENGE (S3.24). */ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s) { @@ -265,6 +267,10 @@ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s } /** Parse reply arguments in the form 'METHODS=COOKIE,SAFECOOKIE COOKIEFILE=".../control_auth_cookie"'. + * Returns a map of keys to values, or an empty map if there was an error. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21), AUTHCHALLENGE (S3.24), + * and ADD_ONION (S3.27). See also sections 2.1 and 2.3. */ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) { @@ -272,28 +278,74 @@ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string size_t ptr=0; while (ptr < s.size()) { std::string key, value; - while (ptr < s.size() && s[ptr] != '=') { + while (ptr < s.size() && s[ptr] != '=' && s[ptr] != ' ') { key.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); + if (s[ptr] == ' ') // The remaining string is an OptArguments + break; ++ptr; // skip '=' if (ptr < s.size() && s[ptr] == '"') { // Quoted string - ++ptr; // skip '=' + ++ptr; // skip opening '"' bool escape_next = false; - while (ptr < s.size() && (!escape_next && s[ptr] != '"')) { - escape_next = (s[ptr] == '\\'); + while (ptr < s.size() && (escape_next || s[ptr] != '"')) { + // Repeated backslashes must be interpreted as pairs + escape_next = (s[ptr] == '\\' && !escape_next); value.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); ++ptr; // skip closing '"' - /* TODO: unescape value - according to the spec this depends on the - * context, some strings use C-LogPrintf style escape codes, some - * don't. So may be better handled at the call site. + /** + * Unescape value. Per https://spec.torproject.org/control-spec section 2.1.1: + * + * For future-proofing, controller implementors MAY use the following + * rules to be compatible with buggy Tor implementations and with + * future ones that implement the spec as intended: + * + * Read \n \t \r and \0 ... \377 as C escapes. + * Treat a backslash followed by any other character as that character. */ + std::string escaped_value; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '\\') { + // This will always be valid, because if the QuotedString + // ended in an odd number of backslashes, then the parser + // would already have returned above, due to a missing + // terminating double-quote. + ++i; + if (value[i] == 'n') { + escaped_value.push_back('\n'); + } else if (value[i] == 't') { + escaped_value.push_back('\t'); + } else if (value[i] == 'r') { + escaped_value.push_back('\r'); + } else if ('0' <= value[i] && value[i] <= '7') { + size_t j; + // Octal escape sequences have a limit of three octal digits, + // but terminate at the first character that is not a valid + // octal digit if encountered sooner. + for (j = 1; j < 3 && (i+j) < value.size() && '0' <= value[i+j] && value[i+j] <= '7'; ++j) {} + // Tor restricts first digit to 0-3 for three-digit octals. + // A leading digit of 4-7 would therefore be interpreted as + // a two-digit octal. + if (j == 3 && value[i] > '3') { + j--; + } + escaped_value.push_back(strtol(value.substr(i, j).c_str(), NULL, 8)); + // Account for automatic incrementing at loop end + i += j - 1; + } else { + escaped_value.push_back(value[i]); + } + } else { + escaped_value.push_back(value[i]); + } + } + value = escaped_value; } else { // Unquoted value. Note that values can contain '=' at will, just no spaces while (ptr < s.size() && s[ptr] != ' ') { value.push_back(s[ptr]); @@ -323,6 +375,10 @@ static std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size char buffer[128]; size_t n; while ((n=fread(buffer, 1, sizeof(buffer), f)) > 0) { + // Check for reading errors so we don't return any data if we couldn't + // read the entire file (or up to maxsize) + if (ferror(f)) + return std::make_pair(false,""); retval.append(buffer, buffer+n); if (retval.size() > maxsize) break; @@ -439,6 +495,13 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe if ((i = m.find("PrivateKey")) != m.end()) private_key = i->second; } + if (service_id.empty()) { + LogPrintf("tor: Error parsing ADD_ONION parameters:\n"); + for (const std::string &s : reply.lines) { + LogPrintf(" %s\n", SanitizeString(s)); + } + return; + } service = LookupNumeric(std::string(service_id+".onion").c_str(), GetListenPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { @@ -516,6 +579,10 @@ void TorController::authchallenge_cb(TorControlConnection& _conn, const TorContr std::pair<std::string,std::string> l = SplitTorReplyLine(reply.lines[0]); if (l.first == "AUTHCHALLENGE") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); + if (m.empty()) { + LogPrintf("tor: Error parsing AUTHCHALLENGE parameters: %s\n", SanitizeString(l.second)); + return; + } std::vector<uint8_t> serverHash = ParseHex(m["SERVERHASH"]); std::vector<uint8_t> serverNonce = ParseHex(m["SERVERNONCE"]); LogPrint(BCLog::TOR, "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); diff --git a/src/txdb.cpp b/src/txdb.cpp index a3889fdf79..c8f5090293 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include <boost/thread.hpp> +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,17 +25,40 @@ static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinEntry { + COutPoint* outpoint; + char key; + CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {} + + template<typename Stream> + void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template<typename Stream> + void Unserialize(Stream& s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; + +} CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -50,10 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) + batch.Erase(entry); else - batch.Write(std::make_pair(DB_COINS, it->first), it->second.coins); + batch.Write(entry, it->second.coin); changed++; } count++; @@ -63,8 +88,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint(BCLog::COINDB, "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); - return db.WriteBatch(batch); + bool ret = db.WriteBatch(batch); + LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); + return ret; +} + +size_t CCoinsViewDB::EstimateSize() const +{ + return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -96,25 +127,31 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (i->pcursor->Valid()) { + CoinEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; + } else { + i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false + } return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { - return pcursor->GetValue(coins); + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const @@ -124,14 +161,18 @@ unsigned int CCoinsViewDBCursor::GetValueSize() const bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { @@ -169,7 +210,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } -bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex) +bool CBlockTreeDB::LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex) { std::unique_ptr<CDBIterator> pcursor(NewIterator()); @@ -211,3 +252,103 @@ bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256 return true; } + +namespace { + +//! Legacy class to deserialize pre-pertxout database entries without reindex. +class CCoins +{ +public: + //! whether transaction is a coinbase + bool fCoinBase; + + //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped + std::vector<CTxOut> vout; + + //! at which height this transaction was included in the active block chain + int nHeight; + + //! empty constructor + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } + + template<typename Stream> + void Unserialize(Stream &s) { + unsigned int nCode = 0; + // version + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + // header code + ::Unserialize(s, VARINT(nCode)); + fCoinBase = nCode & 1; + std::vector<bool> vAvail(2, false); + vAvail[0] = (nCode & 2) != 0; + vAvail[1] = (nCode & 4) != 0; + unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); + // spentness bitmask + while (nMaskCode > 0) { + unsigned char chAvail = 0; + ::Unserialize(s, chAvail); + for (unsigned int p = 0; p < 8; p++) { + bool f = (chAvail & (1 << p)) != 0; + vAvail.push_back(f); + } + if (chAvail != 0) + nMaskCode--; + } + // txouts themself + vout.assign(vAvail.size(), CTxOut()); + for (unsigned int i = 0; i < vAvail.size(); i++) { + if (vAvail[i]) + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); + } + // coinbase height + ::Unserialize(s, VARINT(nHeight)); + } +}; + +} + +/** Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr<CDBIterator> pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair<unsigned char, uint256> key; + if (pcursor->GetKey(key) && key.first == DB_COINS) { + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + pcursor->Next(); + } else { + break; + } + } + db.WriteBatch(batch); + return true; +} diff --git a/src/txdb.h b/src/txdb.h index d9214ba618..974dd4ebe3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -15,8 +15,6 @@ #include <utility> #include <vector> -#include <boost/function.hpp> - class CBlockIndex; class CCoinsViewDBCursor; class uint256; @@ -24,9 +22,7 @@ class uint256; //! Compensate for extra memory peak (x1.5-x1.9) at flush time. static constexpr int DB_PEAK_USAGE_FACTOR = 2; //! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; -//! Always periodic flush if less than this much space still available. -static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; +static constexpr int MAX_BLOCK_COINSDB_USAGE = 10 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) @@ -75,11 +71,15 @@ protected: public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + + //! Attempt to update from an older database format. Returns whether an error occurred. + bool Upgrade(); + size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ @@ -88,8 +88,8 @@ class CCoinsViewDBCursor: public CCoinsViewCursor public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -99,7 +99,7 @@ private: CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr<CDBIterator> pcursor; - std::pair<char, uint256> keyTmp; + std::pair<char, COutPoint> keyTmp; friend class CCoinsViewDB; }; @@ -122,7 +122,7 @@ public: bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &list); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex); + bool LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index ac842da6bf..17389db9f0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -6,6 +6,7 @@ #include "txmempool.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "policy/policy.h" @@ -23,7 +24,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); - nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); + nUsageSize = RecursiveDynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); @@ -342,17 +343,10 @@ CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : nCheckFrequency = 0; } -void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); - - auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); - - // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first->hash == hashTx) { - coins.Spend(it->first->n); // and remove those outputs from coins - it++; - } + return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const @@ -448,7 +442,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -530,9 +524,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); - if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { + const Coin &coin = pcoins->AccessCoin(txin.prevout); + if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; } @@ -660,8 +654,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); - assert(coins && coins->IsAvailable(txin.prevout.n)); + assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -865,6 +858,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD BOOST_FOREACH(txiter descendantIt, setDescendants) { mapTx.modify(descendantIt, update_ancestor_state(0, nFeeDelta, 0, 0)); } + ++nTransactionsUpdated; } } LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); @@ -896,20 +890,24 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } else { + return false; + } } - return (base->GetCoins(txid, coins) && !coins.IsPruned()); + return (base->GetCoin(outpoint, coin) && !coin.IsSpent()); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins(txid); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1020,7 +1018,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { } } -void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining) { +void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; @@ -1051,11 +1049,10 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (exists(txin.prevout.hash)) - continue; - auto iter = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || iter->first->hash != txin.prevout.hash) - pvNoSpendsRemaining->push_back(txin.prevout.hash); + if (exists(txin.prevout.hash)) continue; + if (!mapNextTx.count(txin.prevout)) { + pvNoSpendsRemaining->push_back(txin.prevout); + } } } } @@ -1072,3 +1069,5 @@ bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLi return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 92c4d9f9d4..0316b42ba2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,6 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "policy/feerate.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -23,14 +24,15 @@ #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" #include "boost/multi_index/hashed_index.hpp" +#include <boost/multi_index/sequenced_index.hpp> #include <boost/signals2/signal.hpp> class CAutoFile; class CBlockIndex; -/** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ -static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; +/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ +static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { @@ -59,11 +61,6 @@ class CTxMemPool; * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * - * If updating the descendant state is skipped, we can mark the entry as - * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal nTxSize/ - * nFee+feeDelta. (This can potentially happen during a reorg, where we limit the - * amount of work we're willing to do to avoid consuming too much CPU.) - * */ class CTxMemPoolEntry @@ -82,9 +79,7 @@ private: // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these - // descendants as well. if nCountWithDescendants is 0, treat this entry as - // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be - // correct. + // descendants as well. uint64_t nCountWithDescendants; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) @@ -115,7 +110,7 @@ public: size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } - // Adjusts the descendant state, if this entry is not dirty. + // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); @@ -191,7 +186,7 @@ private: const LockPoints& lp; }; -// extracts a TxMemPoolEntry's transaction hash +// extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -199,6 +194,11 @@ struct mempoolentry_txid { return entry.GetTx().GetHash(); } + + result_type operator() (const CTransactionRef& tx) const + { + return tx->GetHash(); + } }; /** \class CompareTxMemPoolEntryByDescendantScore @@ -327,6 +327,20 @@ enum class MemPoolRemovalReason { REPLACED //! Removed for replacement }; +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -398,20 +412,12 @@ enum class MemPoolRemovalReason { * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * - * Adding transactions from a disconnected block can be very time consuming, - * because we don't have a way to limit the number of in-mempool descendants. - * To bound CPU processing, we limit the amount of work we're willing to do - * to properly update the descendant information for a tx being added from - * a disconnected block. If we would exceed the limit, then we instead mark - * the entry as "dirty", and set the feerate for sorting purposes to be equal - * the feerate of the transaction without any descendants. - * */ class CTxMemPool { private: uint32_t nCheckFrequency; //!< Value n means that n times in 2^32 we check. - unsigned int nTransactionsUpdated; + unsigned int nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. @@ -523,7 +529,7 @@ public: void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector<uint256>& vtxid); - void pruneSpent(const uint256& hash, CCoins &coins); + bool isSpent(const COutPoint& outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** @@ -584,10 +590,10 @@ public: CFeeRate GetMinFee(size_t sizelimit) const; /** Remove transactions from the mempool until its dynamic size is <= sizelimit. - * pvNoSpendsRemaining, if set, will be populated with the list of transactions + * pvNoSpendsRemaining, if set, will be populated with the list of outpoints * which are not in mempool which no longer have any spends in this mempool. */ - void TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining=NULL); + void TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining=NULL); /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); @@ -613,6 +619,13 @@ public: return (mapTx.count(hash) != 0); } + bool exists(const COutPoint& outpoint) const + { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); + } + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; @@ -672,8 +685,99 @@ protected: public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; +}; + +/** + * DisconnectedBlockTransactions + + * During the reorg, it's desirable to re-add previously confirmed transactions + * to the mempool, so that anything not re-confirmed in the new chain is + * available to be mined. However, it's more efficient to wait until the reorg + * is complete and process all still-unconfirmed transactions at that time, + * since we expect most confirmed transactions to (typically) still be + * confirmed in the new chain, and re-accepting to the memory pool is expensive + * (and therefore better to not do in the middle of reorg-processing). + * Instead, store the disconnected transactions (in order!) as we go, remove any + * that are included in blocks in the new chain, and then process the remaining + * still-unconfirmed transactions at the end. + */ + +// multi_index tag names +struct txid_index {}; +struct insertion_order {}; + +struct DisconnectedBlockTransactions { + typedef boost::multi_index_container< + CTransactionRef, + boost::multi_index::indexed_by< + // sorted by txid + boost::multi_index::hashed_unique< + boost::multi_index::tag<txid_index>, + mempoolentry_txid, + SaltedTxidHasher + >, + // sorted by order in the blockchain + boost::multi_index::sequenced< + boost::multi_index::tag<insertion_order> + > + > + > indexed_disconnected_transactions; + + // It's almost certainly a logic bug if we don't clear out queuedTx before + // destruction, as we add to it while disconnecting blocks, and then we + // need to re-process remaining transactions to ensure mempool consistency. + // For now, assert() that we've emptied out this object on destruction. + // This assert() can always be removed if the reorg-processing code were + // to be refactored such that this assumption is no longer true (for + // instance if there was some other way we cleaned up the mempool after a + // reorg, besides draining this object). + ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } + + indexed_disconnected_transactions queuedTx; + uint64_t cachedInnerUsage = 0; + + // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as + // no exact formula for boost::multi_index_contained is implemented. + size_t DynamicMemoryUsage() const { + return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void*)) * queuedTx.size() + cachedInnerUsage; + } + + void addTransaction(const CTransactionRef& tx) + { + queuedTx.insert(tx); + cachedInnerUsage += RecursiveDynamicUsage(tx); + } + + // Remove entries based on txid_index, and update memory usage. + void removeForBlock(const std::vector<CTransactionRef>& vtx) + { + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (auto const &tx : vtx) { + auto it = queuedTx.find(tx->GetHash()); + if (it != queuedTx.end()) { + cachedInnerUsage -= RecursiveDynamicUsage(*it); + queuedTx.erase(it); + } + } + } + + // Remove an entry by insertion_order index, and update memory usage. + void removeEntry(indexed_disconnected_transactions::index<insertion_order>::type::iterator entry) + { + cachedInnerUsage -= RecursiveDynamicUsage(*entry); + queuedTx.get<insertion_order>().erase(entry); + } + + void clear() + { + cachedInnerUsage = 0; + queuedTx.clear(); + } }; #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/undo.h b/src/undo.h index a94e31f5cc..3749d5d7a8 100644 --- a/src/undo.h +++ b/src/undo.h @@ -7,58 +7,90 @@ #define BITCOIN_UNDO_H #include "compressor.h" +#include "consensus/consensus.h" #include "primitives/transaction.h" #include "serialize.h" /** Undo information for a CTxIn * - * Contains the prevout's CTxOut being spent, and if this was the - * last output of the affected transaction, its metadata as well - * (coinbase or not, height, transaction version) + * Contains the prevout's CTxOut being spent, and its metadata as well + * (coinbase or not, height). The serialization contains a dummy value of + * zero. This is be compatible with older versions which expect to see + * the transaction version there. */ -class CTxInUndo +class TxInUndoSerializer { -public: - CTxOut txout; // the txout data before being spent - bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase - unsigned int nHeight; // if the outpoint was the last unspent: its height - int nVersion; // if the outpoint was the last unspent: its version - - CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } + const Coin* txout; +public: template<typename Stream> void Serialize(Stream &s) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion)); - ::Serialize(s, CTxOutCompressor(REF(txout))); + ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1 : 0))); + if (txout->nHeight > 0) { + // Required to maintain compatibility with older undo format. + ::Serialize(s, (unsigned char)0); + } + ::Serialize(s, CTxOutCompressor(REF(txout->out))); } + TxInUndoSerializer(const Coin* coin) : txout(coin) {} +}; + +class TxInUndoDeserializer +{ + Coin* txout; + +public: template<typename Stream> void Unserialize(Stream &s) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - nHeight = nCode / 2; - fCoinBase = nCode & 1; - if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion)); - ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); + txout->nHeight = nCode / 2; + txout->fCoinBase = nCode & 1; + if (txout->nHeight > 0) { + // Old versions stored the version number for the last spend of + // a transaction's outputs. Non-final spends were indicated with + // height = 0. + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + } + ::Unserialize(s, REF(CTxOutCompressor(REF(txout->out)))); } + + TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); + /** Undo information for a CTransaction */ class CTxUndo { public: // undo information for all txins - std::vector<CTxInUndo> vprevout; + std::vector<Coin> vprevout; - ADD_SERIALIZE_METHODS; + template <typename Stream> + void Serialize(Stream& s) const { + // TODO: avoid reimplementing vector serializer + uint64_t count = vprevout.size(); + ::Serialize(s, COMPACTSIZE(REF(count))); + for (const auto& prevout : vprevout) { + ::Serialize(s, REF(TxInUndoSerializer(&prevout))); + } + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vprevout); + template <typename Stream> + void Unserialize(Stream& s) { + // TODO: avoid reimplementing vector deserializer + uint64_t count = 0; + ::Unserialize(s, COMPACTSIZE(count)); + if (count > MAX_INPUTS_PER_BLOCK) { + throw std::ios_base::failure("Too many input undo records"); + } + vprevout.resize(count); + for (auto& prevout : vprevout) { + ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); + } } }; diff --git a/src/util.cpp b/src/util.cpp index cf10ee4aa5..653a4f072a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -13,7 +13,6 @@ #include "fs.h" #include "random.h" #include "serialize.h" -#include "sync.h" #include "utilstrencodings.h" #include "utiltime.h" @@ -92,10 +91,7 @@ const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; -CCriticalSection cs_args; -std::map<std::string, std::string> mapArgs; -static std::map<std::string, std::vector<std::string> > _mapMultiArgs; -const std::map<std::string, std::vector<std::string> >& mapMultiArgs = _mapMultiArgs; +ArgsManager gArgs; bool fPrintToConsole = false; bool fPrintToDebugLog = true; @@ -310,10 +306,14 @@ static std::string LogTimestampStr(const std::string &str, std::atomic_bool *fSt return str; if (*fStartedNewLine) { - int64_t nTimeMicros = GetLogTimeMicros(); + int64_t nTimeMicros = GetTimeMicros(); strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros/1000000); if (fLogTimeMicros) strStamped += strprintf(".%06d", nTimeMicros%1000000); + int64_t mocktime = GetMockTime(); + if (mocktime) { + strStamped += " (mocktime: " + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", mocktime) + ")"; + } strStamped += ' ' + str; } else strStamped = str; @@ -384,11 +384,11 @@ static void InterpretNegativeSetting(std::string& strKey, std::string& strValue) } } -void ParseParameters(int argc, const char* const argv[]) +void ArgsManager::ParseParameters(int argc, const char* const argv[]) { LOCK(cs_args); mapArgs.clear(); - _mapMultiArgs.clear(); + mapMultiArgs.clear(); for (int i = 1; i < argc; i++) { @@ -416,17 +416,23 @@ void ParseParameters(int argc, const char* const argv[]) InterpretNegativeSetting(str, strValue); mapArgs[str] = strValue; - _mapMultiArgs[str].push_back(strValue); + mapMultiArgs[str].push_back(strValue); } } -bool IsArgSet(const std::string& strArg) +std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) +{ + LOCK(cs_args); + return mapMultiArgs.at(strArg); +} + +bool ArgsManager::IsArgSet(const std::string& strArg) { LOCK(cs_args); return mapArgs.count(strArg); } -std::string GetArg(const std::string& strArg, const std::string& strDefault) +std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -434,7 +440,7 @@ std::string GetArg(const std::string& strArg, const std::string& strDefault) return strDefault; } -int64_t GetArg(const std::string& strArg, int64_t nDefault) +int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -442,7 +448,7 @@ int64_t GetArg(const std::string& strArg, int64_t nDefault) return nDefault; } -bool GetBoolArg(const std::string& strArg, bool fDefault) +bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -450,16 +456,16 @@ bool GetBoolArg(const std::string& strArg, bool fDefault) return fDefault; } -bool SoftSetArg(const std::string& strArg, const std::string& strValue) +bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); if (mapArgs.count(strArg)) return false; - mapArgs[strArg] = strValue; + ForceSetArg(strArg, strValue); return true; } -bool SoftSetBoolArg(const std::string& strArg, bool fValue) +bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) { if (fValue) return SoftSetArg(strArg, std::string("1")); @@ -467,10 +473,11 @@ bool SoftSetBoolArg(const std::string& strArg, bool fValue) return SoftSetArg(strArg, std::string("0")); } -void ForceSetArg(const std::string& strArg, const std::string& strValue) +void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); mapArgs[strArg] = strValue; + mapMultiArgs[strArg].push_back(strValue); } @@ -589,7 +596,7 @@ fs::path GetConfigFile(const std::string& confPath) return pathConfigFile; } -void ReadConfigFile(const std::string& confPath) +void ArgsManager::ReadConfigFile(const std::string& confPath) { fs::ifstream streamConfig(GetConfigFile(confPath)); if (!streamConfig.good()) @@ -608,7 +615,7 @@ void ReadConfigFile(const std::string& confPath) InterpretNegativeSetting(strKey, strValue); if (mapArgs.count(strKey) == 0) mapArgs[strKey] = strValue; - _mapMultiArgs[strKey].push_back(strValue); + mapMultiArgs[strKey].push_back(strValue); } } // If datadir is changed in .conf file: diff --git a/src/util.h b/src/util.h index ed28070a3f..4386ddd550 100644 --- a/src/util.h +++ b/src/util.h @@ -16,6 +16,7 @@ #include "compat.h" #include "fs.h" +#include "sync.h" #include "tinyformat.h" #include "utiltime.h" @@ -41,7 +42,6 @@ public: boost::signals2::signal<std::string (const char* psz)> Translate; }; -extern const std::map<std::string, std::vector<std::string> >& mapMultiArgs; extern bool fPrintToConsole; extern bool fPrintToDebugLog; @@ -148,7 +148,6 @@ bool error(const char* fmt, const Args&... args) } void PrintExceptionContinue(const std::exception *pex, const char* pszThread); -void ParseParameters(int argc, const char*const argv[]); void FileCommit(FILE *file); bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); @@ -163,7 +162,6 @@ fs::path GetConfigFile(const std::string& confPath); fs::path GetPidFile(); void CreatePidFile(const fs::path &path, pid_t pid); #endif -void ReadConfigFile(const std::string& confPath); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif @@ -180,6 +178,16 @@ inline bool IsSwitchChar(char c) #endif } +class ArgsManager +{ +protected: + CCriticalSection cs_args; + std::map<std::string, std::string> mapArgs; + std::map<std::string, std::vector<std::string> > mapMultiArgs; +public: + void ParseParameters(int argc, const char*const argv[]); + void ReadConfigFile(const std::string& confPath); + std::vector<std::string> GetArgs(const std::string& strArg); /** * Return true if the given argument has been manually set * @@ -233,8 +241,58 @@ bool SoftSetArg(const std::string& strArg, const std::string& strValue); */ bool SoftSetBoolArg(const std::string& strArg, bool fValue); -// Forces a arg setting, used only in testing +// Forces an arg setting. Called by SoftSetArg() if the arg hasn't already +// been set. Also called directly in testing. void ForceSetArg(const std::string& strArg, const std::string& strValue); +}; + +extern ArgsManager gArgs; + +// wrappers using the global ArgsManager: +static inline void ParseParameters(int argc, const char*const argv[]) +{ + gArgs.ParseParameters(argc, argv); +} + +static inline void ReadConfigFile(const std::string& confPath) +{ + gArgs.ReadConfigFile(confPath); +} + +static inline bool SoftSetArg(const std::string& strArg, const std::string& strValue) +{ + return gArgs.SoftSetArg(strArg, strValue); +} + +static inline void ForceSetArg(const std::string& strArg, const std::string& strValue) +{ + gArgs.ForceSetArg(strArg, strValue); +} + +static inline bool IsArgSet(const std::string& strArg) +{ + return gArgs.IsArgSet(strArg); +} + +static inline std::string GetArg(const std::string& strArg, const std::string& strDefault) +{ + return gArgs.GetArg(strArg, strDefault); +} + +static inline int64_t GetArg(const std::string& strArg, int64_t nDefault) +{ + return gArgs.GetArg(strArg, nDefault); +} + +static inline bool GetBoolArg(const std::string& strArg, bool fDefault) +{ + return gArgs.GetBoolArg(strArg, fDefault); +} + +static inline bool SoftSetBoolArg(const std::string& strArg, bool fValue) +{ + return gArgs.SoftSetBoolArg(strArg, fValue); +} /** * Format a string to be used as group of options in help messages diff --git a/src/utiltime.cpp b/src/utiltime.cpp index 510f540b1d..e07069125d 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -31,6 +31,11 @@ void SetMockTime(int64_t nMockTimeIn) nMockTime.store(nMockTimeIn, std::memory_order_relaxed); } +int64_t GetMockTime() +{ + return nMockTime.load(std::memory_order_relaxed); +} + int64_t GetTimeMillis() { int64_t now = (boost::posix_time::microsec_clock::universal_time() - @@ -52,15 +57,6 @@ int64_t GetSystemTimeInSeconds() return GetTimeMicros()/1000000; } -/** Return a time useful for the debug log */ -int64_t GetLogTimeMicros() -{ - int64_t mocktime = nMockTime.load(std::memory_order_relaxed); - if (mocktime) return mocktime*1000000; - - return GetTimeMicros(); -} - void MilliSleep(int64_t n) { diff --git a/src/utiltime.h b/src/utiltime.h index cc3290c631..8ae8540b89 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -23,8 +23,8 @@ int64_t GetTime(); int64_t GetTimeMillis(); int64_t GetTimeMicros(); int64_t GetSystemTimeInSeconds(); // Like GetTime(), but not mockable -int64_t GetLogTimeMicros(); void SetMockTime(int64_t nMockTimeIn); +int64_t GetMockTime(); void MilliSleep(int64_t n); std::string DateTimeStrFormat(const char* pszFormat, int64_t nTime); diff --git a/src/validation.cpp b/src/validation.cpp index f189b741cd..de65839eef 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -12,6 +12,7 @@ #include "checkqueue.h" #include "consensus/consensus.h" #include "consensus/merkle.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "fs.h" #include "hash.h" @@ -188,19 +189,6 @@ enum FlushStateMode { bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0); void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) -{ - if (tx.nLockTime == 0) - return true; - if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) - return true; - for (const auto& txin : tx.vin) { - if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) - return false; - } - return true; -} - bool CheckFinalTx(const CTransaction &tx, int flags) { AssertLockHeld(cs_main); @@ -233,89 +221,6 @@ bool CheckFinalTx(const CTransaction &tx, int flags) return IsFinalTx(tx, nBlockHeight, nBlockTime); } -/** - * Calculates the block height and previous block's median time past at - * which the transaction will be considered final in the context of BIP 68. - * Also removes from the vector of input heights any entries which did not - * correspond to sequence locked inputs as they do not affect the calculation. - */ -static std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - assert(prevHeights->size() == tx.vin.size()); - - // Will be set to the equivalent height- and time-based nLockTime - // values that would be necessary to satisfy all relative lock- - // time constraints given our view of block chain history. - // The semantics of nLockTime are the last invalid height/time, so - // use -1 to have the effect of any height or time being valid. - int nMinHeight = -1; - int64_t nMinTime = -1; - - // tx.nVersion is signed integer so requires cast to unsigned otherwise - // we would be doing a signed comparison and half the range of nVersion - // wouldn't support BIP 68. - bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 - && flags & LOCKTIME_VERIFY_SEQUENCE; - - // Do not enforce sequence numbers as a relative lock time - // unless we have been instructed to - if (!fEnforceBIP68) { - return std::make_pair(nMinHeight, nMinTime); - } - - for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { - const CTxIn& txin = tx.vin[txinIndex]; - - // Sequence numbers with the most significant bit set are not - // treated as relative lock-times, nor are they given any - // consensus-enforced meaning at this point. - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { - // The height of this input is not relevant for sequence locks - (*prevHeights)[txinIndex] = 0; - continue; - } - - int nCoinHeight = (*prevHeights)[txinIndex]; - - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { - int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); - // NOTE: Subtract 1 to maintain nLockTime semantics - // BIP 68 relative lock times have the semantics of calculating - // the first block or time at which the transaction would be - // valid. When calculating the effective block time or height - // for the entire transaction, we switch to using the - // semantics of nLockTime which is the last invalid block - // time or height. Thus we subtract 1 from the calculated - // time or height. - - // Time-based relative lock-times are measured from the - // smallest allowed timestamp of the block containing the - // txout being spent, which is the median time past of the - // block prior. - nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); - } else { - nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); - } - } - - return std::make_pair(nMinHeight, nMinTime); -} - -static bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) -{ - assert(block.pprev); - int64_t nBlockTime = block.pprev->GetMedianTimePast(); - if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) - return false; - - return true; -} - -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); -} - bool TestLockPointValidity(const LockPoints* lp) { AssertLockHeld(cs_main); @@ -363,15 +268,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -404,117 +309,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool return EvaluateSequenceLocks(index, lockPair); } - -unsigned int GetLegacySigOpCount(const CTransaction& tx) -{ - unsigned int nSigOps = 0; - for (const auto& txin : tx.vin) - { - nSigOps += txin.scriptSig.GetSigOpCount(false); - } - for (const auto& txout : tx.vout) - { - nSigOps += txout.scriptPubKey.GetSigOpCount(false); - } - return nSigOps; -} - -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) -{ - if (tx.IsCoinBase()) - return 0; - - unsigned int nSigOps = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - if (prevout.scriptPubKey.IsPayToScriptHash()) - nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); - } - return nSigOps; -} - -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) -{ - int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; - - if (tx.IsCoinBase()) - return nSigOps; - - if (flags & SCRIPT_VERIFY_P2SH) { - nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; - } - - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); - } - return nSigOps; -} - - - - - -bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) -{ - // Basic checks that don't depend on any context - if (tx.vin.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); - if (tx.vout.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); - - // Check for negative or overflow output values - CAmount nValueOut = 0; - for (const auto& txout : tx.vout) - { - if (txout.nValue < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); - if (txout.nValue > MAX_MONEY) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); - nValueOut += txout.nValue; - if (!MoneyRange(nValueOut)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); - } - - // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock - if (fCheckDuplicateInputs) { - std::set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (!vInOutPoints.insert(txin.prevout).second) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); - } - } - - if (tx.IsCoinBase()) - { - if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); - } - else - { - for (const auto& txin : tx.vin) - if (txin.prevout.IsNull()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); - } - - return true; -} - void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { int expired = pool.Expire(GetTime() - age); if (expired != 0) { LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } - std::vector<uint256> vNoSpendsRemaining; + std::vector<COutPoint> vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining) pcoinsTip->Uncache(removed); } @@ -539,9 +342,59 @@ static bool IsCurrentForFeeEstimation() return true; } +/* Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + +void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool fAddToMempool) +{ + AssertLockHeld(cs_main); + std::vector<uint256> vHashUpdate; + // disconnectpool's insertion_order index sorts the entries from + // oldest to newest, but the oldest entry will be the last tx from the + // latest mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions + // back to the mempool starting with the earliest transaction that had + // been previously seen in a block. + auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); + while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { + // ignore validation errors in resurrected transactions + CValidationState stateDummy; + if (!fAddToMempool || (*it)->IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, *it, false, NULL, NULL, true)) { + // If the transaction doesn't make it in to the mempool, remove any + // transactions that depend on it (which would now be orphans). + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (mempool.exists((*it)->GetHash())) { + vHashUpdate.push_back((*it)->GetHash()); + } + ++it; + } + disconnectpool.queuedTx.clear(); + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in + // the disconnectpool that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + + // We also need to remove any now-immature transactions + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + // Re-limit mempool size, in case we added any transactions + LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); +} + bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -634,30 +487,30 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C view.SetBackend(viewMemPool); // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + for (size_t out = 0; out < tx.vout.size(); out++) { + COutPoint outpoint(hash, out); + bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); + if (view.HaveCoin(outpoint)) { + if (!had_coin_in_cache) { + coins_to_uncache.push_back(outpoint); + } + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } } // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) + if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); + } + if (!view.HaveCoin(txin.prevout)) { + if (pfMissingInputs) { *pfMissingInputs = true; + } return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - // Bring the best block into scope view.GetBestBlock(); @@ -695,8 +548,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { + const Coin &coin = view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } @@ -960,10 +813,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector<COutPoint> coins_to_uncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -1015,15 +868,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus } if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1141,6 +987,7 @@ bool IsInitialBlockDownload() return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; + LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); latchToFalse.store(true, std::memory_order_relaxed); return false; } @@ -1270,24 +1117,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; - } + txundo.vprevout.emplace_back(); + inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1309,52 +1144,6 @@ int GetSpendHeight(const CCoinsViewCache& inputs) return pindexPrev->nHeight + 1; } -namespace Consensus { -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) -{ - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!inputs.HaveInputs(tx)) - return state.Invalid(false, 0, "", "Inputs unavailable"); - - CAmount nValueIn = 0; - CAmount nFees = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { - if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) - return state.Invalid(false, - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); - } - - // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); - - } - - if (nValueIn < tx.GetValueOut()) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, - strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); - - // Tally transaction fees - CAmount nTxFee = nValueIn - tx.GetValueOut(); - if (nTxFee < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); - nFees += nTxFee; - if (!MoneyRange(nFees)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); - return true; -} -}// namespace Consensus - bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) { if (!tx.IsCoinBase()) @@ -1377,11 +1166,19 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + const CScript& scriptPubKey = coin.out.scriptPubKey; + const CAmount amount = coin.out.nValue; // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); + CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); @@ -1393,7 +1190,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, + CScriptCheck check2(scriptPubKey, amount, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); @@ -1452,8 +1249,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << hashBlock; + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1461,10 +1260,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; @@ -1490,61 +1286,66 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // anon namespace +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + view.AddCoin(out, std::move(undo), undo.fCoinBase); - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } /** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -static bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean = NULL) + * When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */ +static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { assert(pindex->GetBlockHash() == view.GetBestBlock()); - if (pfClean) - *pfClean = false; - bool fClean = true; CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) - return error("DisconnectBlock(): no undo data available"); - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock(): failure reading undo data"); + if (pos.IsNull()) { + error("DisconnectBlock(): no undo data available"); + return DISCONNECT_FAILED; + } + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } - if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock(): block and undo data inconsistent"); + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1553,46 +1354,38 @@ static bool DisconnectBlock(const CBlock& block, CValidationState& state, const // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + view.SpendCoin(out, &coin); + if (tx.vout[o] != coin.out) { + fClean = false; // transaction output mismatch + } + } } // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); - if (pfClean) { - *pfClean = fClean; - return true; - } - - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } void static FlushBlockFile(bool fFinalize = false) @@ -1766,10 +1559,12 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (fEnforceBIP30) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + REJECT_INVALID, "bad-txns-BIP30"); + } + } } } @@ -1836,7 +1631,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { @@ -1974,9 +1769,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), - std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -2017,12 +1811,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) @@ -2051,6 +1845,16 @@ void PruneAndFlush() { FlushStateToDisk(state, FLUSH_STATE_NONE); } +static void DoWarning(const std::string& strWarning) +{ + static bool fWarned = false; + SetMiscWarning(strWarning); + if (!fWarned) { + AlertNotify(strWarning); + fWarned = true; + } +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); @@ -2060,7 +1864,6 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { cvBlockChange.notify_all(); - static bool fWarned = false; std::vector<std::string> warningMessages; if (!IsInitialBlockDownload()) { @@ -2070,15 +1873,11 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { + const std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); if (state == THRESHOLD_ACTIVE) { - std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } else { - warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); + warningMessages.push_back(strWarning); } } } @@ -2091,19 +1890,15 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { pindex = pindex->pprev; } if (nUpgraded > 0) - warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); + warningMessages.push_back(strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); if (nUpgraded > 100/2) { std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), @@ -2114,8 +1909,17 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } -/** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) +/** Disconnect chainActive's tip. + * After calling, the mempool will be in an inconsistent state, with + * transactions from disconnected blocks being added to disconnectpool. You + * should make the mempool consistent again by calling UpdateMempoolForReorg. + * with cs_main held. + * + * If disconnectpool is NULL, then no disconnected transactions are added to + * disconnectpool (note that the caller is responsible for mempool consistency + * in any case). + */ +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2128,7 +1932,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); @@ -2138,25 +1942,17 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - if (!fBare) { - // Resurrect mempool transactions from the disconnected block. - std::vector<uint256> vHashUpdate; - for (const auto& it : block.vtx) { - const CTransaction& tx = *it; - // ignore validation errors in resurrected transactions - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { - mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } + if (disconnectpool) { + // Save transactions to re-add to mempool at end of reorg + for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { + disconnectpool->addTransaction(*it); + } + while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { + // Drop the earliest entry, and remove its children from the mempool. + auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + disconnectpool->removeEntry(it); } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); } // Update chainActive and related variables. @@ -2244,7 +2040,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2286,6 +2082,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); @@ -2379,9 +2176,14 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; + DisconnectedBlockTransactions disconnectpool; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams)) + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // This is likely a fatal error, but keep the mempool consistent, + // just in case. Only remove from the mempool in this case. + UpdateMempoolForReorg(disconnectpool, false); return false; + } fBlocksDisconnected = true; } @@ -2404,7 +2206,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) @@ -2415,6 +2217,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c break; } else { // A system error occurred (disk space, database error, ...). + // Make the mempool consistent with the current tip, just in case + // any observers try to use it before shutdown. + UpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2429,8 +2234,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c } if (fBlocksDisconnected) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // If any blocks were disconnected, disconnectpool may be non empty. Add + // any disconnected transactions back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); } mempool.check(pcoinsTip); @@ -2579,6 +2385,7 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); + DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { CBlockIndex *pindexWalk = chainActive.Tip(); pindexWalk->nStatus |= BLOCK_FAILED_CHILD; @@ -2586,13 +2393,17 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams)) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // It's probably hopeless to try to make the mempool consistent + // here if DisconnectTip failed, but we can try. + UpdateMempoolForReorg(disconnectpool, false); return false; } } - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // DisconnectTip will add transactions to disconnectpool; try to add these + // back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. @@ -2605,7 +2416,6 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C } InvalidChainFound(pindex); - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } @@ -3053,8 +2863,8 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam if (!fHaveWitness) { - for (size_t i = 0; i < block.vtx.size(); i++) { - if (block.vtx[i]->HasWitness()) { + for (const auto& tx : block.vtx) { + if (tx->HasWitness()) { return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); } } @@ -3656,15 +3466,17 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { - bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + DisconnectResult res = DisconnectBlock(block, pindex, coins); + if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } pindexState = pindex->pprev; - if (!fClean) { + if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; - } else + } else { nGoodTransactions += block.vtx.size(); + } } if (ShutdownRequested()) return true; @@ -3717,7 +3529,7 @@ bool RewindBlockIndex(const CChainParams& params) // of the blockchain). break; } - if (!DisconnectTip(state, params, true)) { + if (!DisconnectTip(state, params, NULL)) { return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); } // Occasionally flush state to disk. @@ -4169,6 +3981,12 @@ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::D return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); } +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + LOCK(cs_main); + return VersionBitsStatistics(chainActive.Tip(), params, pos); +} + int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); diff --git a/src/validation.h b/src/validation.h index 24ebf238df..096fd0a9ee 100644 --- a/src/validation.h +++ b/src/validation.h @@ -14,6 +14,7 @@ #include "coins.h" #include "fs.h" #include "protocol.h" // For CMessageHeader::MessageStartChars +#include "policy/feerate.h" #include "script/script_error.h" #include "sync.h" #include "versionbits.h" @@ -69,6 +70,8 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -104,7 +107,7 @@ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; /** Average delay between local address broadcasts in seconds. */ -static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; +static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; /** Average delay between peer address broadcasts in seconds. */ static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; /** Average delay between trickled inventory transmissions in seconds. @@ -131,7 +134,8 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; - +/** Default for -persistmempool */ +static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -mempoolreplacement */ static const bool DEFAULT_ENABLE_REPLACEMENT = true; /** Default for using fee filter */ @@ -337,33 +341,12 @@ std::string FormatStateMessage(const CValidationState &state); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); +/** Get the numerical statistics for the BIP9 state for a given deployment at the current tip. */ +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos); + /** Get the block height at which the BIP9 deployment switched into the state for the block building on the current tip. */ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos); -/** - * Count ECDSA signature operations the old-fashioned (pre-0.6) way - * @return number of sigops this transaction's outputs will produce when spent - * @see CTransaction::FetchInputs - */ -unsigned int GetLegacySigOpCount(const CTransaction& tx); - -/** - * Count ECDSA signature operations in pay-to-script-hash inputs. - * - * @param[in] mapInputs Map of previous transactions that have outputs we're spending - * @return maximum number of sigops required to validate this transaction's inputs - * @see CTransaction::FetchInputs - */ -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& mapInputs); - -/** - * Compute total signature operation cost of a transaction. - * @param[in] tx Transaction for which we are computing the cost - * @param[in] inputs Map of previous transactions that have outputs we're spending - * @param[out] flags Script verification flags - * @return Total signature operation cost of tx - */ -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); /** * Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts) @@ -378,26 +361,6 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); /** Transaction validation functions */ -/** Context-independent validity checks */ -bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); - -namespace Consensus { - -/** - * Check whether all inputs of this transaction are valid (no double spends and amounts) - * This does not modify the UTXO set. This does not check scripts and sigs. - * Preconditions: tx.IsCoinBase() is false. - */ -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); - -} // namespace Consensus - -/** - * Check if transaction is final and can be included in a block with the - * specified height and time. Consensus critical. - */ -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); - /** * Check if transaction will be final in the next block to be created. * @@ -413,12 +376,6 @@ bool CheckFinalTx(const CTransaction &tx, int flags = -1); bool TestLockPointValidity(const LockPoints* lp); /** - * Check if transaction is final per BIP 68 sequence numbers and can be included in a block. - * Consensus critical. Takes as input a list of heights at which tx's inputs (in order) confirmed. - */ -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); - -/** * Check if transaction will be BIP 68 final in the next block to be created. * * Simulates calling SequenceLocks() with data from the tip of the current active chain. @@ -449,8 +406,8 @@ private: public: CScriptCheck(): amount(0), ptxTo(0), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR) {} - CScriptCheck(const CCoins& txFromIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : - scriptPubKey(txFromIn.vout[txToIn.vin[nInIn].prevout.n].scriptPubKey), amount(txFromIn.vout[txToIn.vin[nInIn].prevout.n].nValue), + CScriptCheck(const CScript& scriptPubKeyIn, const CAmount amountIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : + scriptPubKey(scriptPubKeyIn), amount(amountIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn) { } bool operator()(); diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 8a7cce7485..80786233f5 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "versionbits.h" - #include "consensus/params.h" const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { @@ -105,6 +104,36 @@ ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* return state; } +// return the numerical statistics of blocks signalling the specified BIP9 condition in this current period +BIP9Stats AbstractThresholdConditionChecker::GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const +{ + BIP9Stats stats; + + stats.period = Period(params); + stats.threshold = Threshold(params); + + if (pindex == NULL) + return stats; + + // Find beginning of period + const CBlockIndex* pindexEndOfPrevPeriod = pindex->GetAncestor(pindex->nHeight - ((pindex->nHeight + 1) % stats.period)); + stats.elapsed = pindex->nHeight - pindexEndOfPrevPeriod->nHeight; + + // Count from current block to beginning of period + int count = 0; + const CBlockIndex* currentIndex = pindex; + while (pindexEndOfPrevPeriod->nHeight != currentIndex->nHeight){ + if (Condition(currentIndex, params)) + count++; + currentIndex = currentIndex->pprev; + } + + stats.count = count; + stats.possible = (stats.period - stats.threshold ) >= (stats.elapsed - count); + + return stats; +} + int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { const ThresholdState initialState = GetStateFor(pindexPrev, params, cache); @@ -167,6 +196,11 @@ ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus:: return VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, cache.caches[pos]); } +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + return VersionBitsConditionChecker(pos).GetStateStatisticsFor(pindexPrev, params); +} + int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache) { return VersionBitsConditionChecker(pos).GetStateSinceHeightFor(pindexPrev, params, cache.caches[pos]); diff --git a/src/versionbits.h b/src/versionbits.h index 7a929266aa..f1d31ea0af 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -37,6 +37,14 @@ struct BIP9DeploymentInfo { bool gbt_force; }; +struct BIP9Stats { + int period; + int threshold; + int elapsed; + int count; + bool possible; +}; + extern const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[]; /** @@ -51,6 +59,7 @@ protected: virtual int Threshold(const Consensus::Params& params) const =0; public: + BIP9Stats GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const; // Note that the functions below take a pindexPrev as input: they compute information for block B based on its parent. ThresholdState GetStateFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; int GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; @@ -64,6 +73,7 @@ struct VersionBitsCache }; ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); uint32_t VersionBitsMask(const Consensus::Params& params, Consensus::DeploymentPos pos); diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 4e93e929be..cb4719ae90 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include "policy/feerate.h" #include "primitives/transaction.h" #include "wallet/wallet.h" @@ -17,8 +18,6 @@ public: bool fAllowOtherInputs; //! Includes watch only addresses which match the ISMINE_WATCH_SOLVABLE criteria bool fAllowWatchOnly; - //! Minimum absolute fee (not per kilobyte) - CAmount nMinimumTotalFee; //! Override estimated feerate bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true @@ -39,7 +38,6 @@ public: fAllowOtherInputs = false; fAllowWatchOnly = false; setSelected.clear(); - nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; nConfirmTarget = 0; diff --git a/src/wallet/db.h b/src/wallet/db.h index 1a46448cc7..3c6870d169 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -180,22 +180,23 @@ public: Dbt datValue; datValue.set_flags(DB_DBT_MALLOC); int ret = pdb->get(activeTxn, &datKey, &datValue, 0); - memset(datKey.get_data(), 0, datKey.get_size()); - if (datValue.get_data() == NULL) - return false; - - // Unserialize value - try { - CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); - ssValue >> value; - } catch (const std::exception&) { - return false; + memory_cleanse(datKey.get_data(), datKey.get_size()); + bool success = false; + if (datValue.get_data() != NULL) { + // Unserialize value + try { + CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); + ssValue >> value; + success = true; + } catch (const std::exception&) { + // In this case success remains 'false' + } + + // Clear and free memory + memory_cleanse(datValue.get_data(), datValue.get_size()); + free(datValue.get_data()); } - - // Clear and free memory - memset(datValue.get_data(), 0, datValue.get_size()); - free(datValue.get_data()); - return (ret == 0); + return ret == 0 && success; } template <typename K, typename T> @@ -222,8 +223,8 @@ public: int ret = pdb->put(activeTxn, &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); // Clear memory in case it was a private key - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); return (ret == 0); } @@ -245,7 +246,7 @@ public: int ret = pdb->del(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0 || ret == DB_NOTFOUND); } @@ -265,7 +266,7 @@ public: int ret = pdb->exists(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0); } @@ -308,8 +309,8 @@ public: ssValue.write((char*)datValue.get_data(), datValue.get_size()); // Clear and free memory - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); free(datKey.get_data()); free(datValue.get_data()); return 0; diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index f5b63c1ecd..99120d290c 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -41,6 +41,31 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWal return GetVirtualTransactionSize(txNew); } +bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx) { + if (pWallet->HasWalletSpend(wtx.GetHash())) { + vErrors.push_back("Transaction has descendants in the wallet"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + + { + LOCK(mempool.cs); + auto it_mp = mempool.mapTx.find(wtx.GetHash()); + if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + vErrors.push_back("Transaction has descendants in the mempool"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + } + + if (wtx.GetDepthInMainChain() != 0) { + vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + currentResult = BumpFeeResult::WALLET_ERROR; + return false; + } + return true; +} + CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable) : txid(std::move(txidIn)), @@ -58,25 +83,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf auto it = pWallet->mapWallet.find(txid); const CWalletTx& wtx = it->second; - if (pWallet->HasWalletSpend(txid)) { - vErrors.push_back("Transaction has descendants in the wallet"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; - } - - { - LOCK(mempool.cs); - auto it_mp = mempool.mapTx.find(txid); - if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { - vErrors.push_back("Transaction has descendants in the mempool"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; - } - } - - if (wtx.GetDepthInMainChain() != 0) { - vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); - currentResult = BumpFeeResult::WALLET_ERROR; + if (!preconditionChecks(pWallet, wtx)) { return; } @@ -214,7 +221,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; - if (poutput->nValue <= poutput->GetDustThreshold(::dustRelayFee)) { + if (poutput->nValue <= GetDustThreshold(*poutput, ::dustRelayFee)) { LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); nNewFee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); @@ -248,6 +255,11 @@ bool CFeeBumper::commit(CWallet *pWallet) } CWalletTx& oldWtx = pWallet->mapWallet[txid]; + // make sure the transaction still has no descendants and hasn't been mined in the meantime + if (!preconditionChecks(pWallet, oldWtx)) { + return false; + } + CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); // commit/broadcast the tx CReserveKey reservekey(pWallet); diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 681f55e4e5..f40d05da28 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -8,6 +8,7 @@ #include <primitives/transaction.h> class CWallet; +class CWalletTx; class uint256; enum class BumpFeeResult @@ -44,6 +45,8 @@ public: bool commit(CWallet *pWalletNonConst); private: + bool preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx); + const uint256 txid; uint256 bumpedTxid; CMutableTransaction mtx; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 82708dab26..613d5d228a 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -357,7 +357,7 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "removeprunedfunds \"txid\"\n" - "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will effect wallet balances.\n" + "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will affect wallet balances.\n" "\nArguments:\n" "1. \"txid\" (string, required) The hex-encoded id of the transaction you are deleting\n" "\nExamples:\n" @@ -536,14 +536,11 @@ UniValue importwallet(const JSONRPCRequest& request) } file.close(); pwallet->ShowProgress("", 100); // hide progress dialog in GUI - - CBlockIndex *pindex = chainActive.Tip(); - while (pindex && pindex->pprev && pindex->GetBlockTime() > nTimeBegin - TIMESTAMP_WINDOW) - pindex = pindex->pprev; - pwallet->UpdateTimeFirstKey(nTimeBegin); - LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); + CBlockIndex *pindex = chainActive.FindEarliestAtLeast(nTimeBegin - TIMESTAMP_WINDOW); + + LogPrintf("Rescanning last %i blocks\n", pindex ? chainActive.Height() - pindex->nHeight + 1 : 0); pwallet->ScanForWalletTransactions(pindex); pwallet->MarkDirty(); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index d1e7485d04..0860d3565c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -12,6 +12,7 @@ #include "wallet/coincontrol.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" #include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" @@ -729,7 +730,8 @@ UniValue getbalance(const JSONRPCRequest& request) if (request.params.size() == 0) return ValueFromAmount(pwallet->GetBalance()); - const std::string* account = request.params[0].get_str() != "*" ? &request.params[0].get_str() : nullptr; + const std::string& account_param = request.params[0].get_str(); + const std::string* account = account_param != "*" ? &account_param : nullptr; int nMinDepth = 1; if (request.params.size() > 1) @@ -2468,22 +2470,29 @@ UniValue listunspent(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() > 4) + if (request.fHelp || request.params.size() > 5) throw std::runtime_error( - "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] )\n" + "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] [query_options])\n" "\nReturns array of unspent transaction outputs\n" "with between minconf and maxconf (inclusive) confirmations.\n" "Optionally filter to only include txouts paid to specified addresses.\n" "\nArguments:\n" "1. minconf (numeric, optional, default=1) The minimum confirmations to filter\n" "2. maxconf (numeric, optional, default=9999999) The maximum confirmations to filter\n" - "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" + "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" " [\n" - " \"address\" (string) bitcoin address\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ]\n" "4. include_unsafe (bool, optional, default=true) Include outputs that are not safe to spend\n" " See description of \"safe\" attribute below.\n" + "5. query_options (json, optional) JSON with query options\n" + " {\n" + " \"minimumAmount\" (numeric or string, default=0) Minimum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumAmount\" (numeric or string, default=unlimited) Maximum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumCount\" (numeric or string, default=unlimited) Maximum number of UTXOs\n" + " \"minimumSumAmount\" (numeric or string, default=unlimited) Minimum sum value of all UTXOs in " + CURRENCY_UNIT + "\n" + " }\n" "\nResult\n" "[ (array of json object)\n" " {\n" @@ -2508,6 +2517,8 @@ UniValue listunspent(const JSONRPCRequest& request) + HelpExampleCli("listunspent", "") + HelpExampleCli("listunspent", "6 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + HelpExampleRpc("listunspent", "6, 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + HelpExampleCli("listunspent", "6 9999999 '[]' true '{ \"minimumAmount\": 0.005 }'") + + HelpExampleRpc("listunspent", "6, 9999999, [] , true, { \"minimumAmount\": 0.005 } ") ); int nMinDepth = 1; @@ -2543,15 +2554,34 @@ UniValue listunspent(const JSONRPCRequest& request) include_unsafe = request.params[3].get_bool(); } + CAmount nMinimumAmount = 0; + CAmount nMaximumAmount = MAX_MONEY; + CAmount nMinimumSumAmount = MAX_MONEY; + uint64_t nMaximumCount = 0; + + if (request.params.size() > 4) { + const UniValue& options = request.params[4].get_obj(); + + if (options.exists("minimumAmount")) + nMinimumAmount = AmountFromValue(options["minimumAmount"]); + + if (options.exists("maximumAmount")) + nMaximumAmount = AmountFromValue(options["maximumAmount"]); + + if (options.exists("minimumSumAmount")) + nMinimumSumAmount = AmountFromValue(options["minimumSumAmount"]); + + if (options.exists("maximumCount")) + nMaximumCount = options["maximumCount"].get_int64(); + } + UniValue results(UniValue::VARR); std::vector<COutput> vecOutputs; assert(pwallet != NULL); LOCK2(cs_main, pwallet->cs_wallet); - pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, true); - BOOST_FOREACH(const COutput& out, vecOutputs) { - if (out.nDepth < nMinDepth || out.nDepth > nMaxDepth) - continue; + pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); + BOOST_FOREACH(const COutput& out, vecOutputs) { CTxDestination address; const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); @@ -2767,7 +2797,7 @@ UniValue bumpfee(const JSONRPCRequest& request) "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" "By default, the new fee will be calculated automatically using estimatefee.\n" "The user can specify a confirmation target for estimatefee.\n" - "Alternatively, the user can specify totalFee, or use RPC setpaytxfee to set a higher fee rate.\n" + "Alternatively, the user can specify totalFee, or use RPC settxfee to set a higher fee rate.\n" "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" "returned by getnetworkinfo) to enter the node's mempool.\n" "\nArguments:\n" @@ -2879,7 +2909,7 @@ UniValue bumpfee(const JSONRPCRequest& request) UniValue errors(UniValue::VARR); for (const std::string& err: feeBump.getErrors()) errors.push_back(err); - result.push_back(errors); + result.push_back(Pair("errors", errors)); return result; } @@ -2934,7 +2964,7 @@ static const CRPCCommand commands[] = { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, false, {"minconf","include_empty","include_watchonly"} }, { "wallet", "listsinceblock", &listsinceblock, false, {"blockhash","target_confirmations","include_watchonly"} }, { "wallet", "listtransactions", &listtransactions, false, {"account","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe"} }, + { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, { "wallet", "lockunspent", &lockunspent, true, {"unlock","transactions"} }, { "wallet", "move", &movecmd, false, {"fromaccount","toaccount","amount","minconf","comment"} }, { "wallet", "sendfrom", &sendfrom, false, {"fromaccount","toaddress","amount","minconf","comment","comment_to"} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 8eeba72a06..5c7359fdce 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -9,6 +9,7 @@ #include <utility> #include <vector> +#include "consensus/validation.h" #include "rpc/server.h" #include "test/test_bitcoin.h" #include "validation.h" @@ -19,6 +20,8 @@ #include <univalue.h> extern UniValue importmulti(const JSONRPCRequest& request); +extern UniValue dumpwallet(const JSONRPCRequest& request); +extern UniValue importwallet(const JSONRPCRequest& request); // how many times to run all the tests to have a chance to catch errors that only show up with particular random shuffles #define RUN_TESTS 100 @@ -437,6 +440,66 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) } } +// Verify importwallet RPC starts rescan at earliest block with timestamp +// greater or equal than key birthday. Previously there was a bug where +// importwallet RPC would start the scan at the latest block with timestamp less +// than or equal to key birthday. +BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) +{ + CWallet *pwalletMainBackup = ::pwalletMain; + LOCK(cs_main); + + // Create two blocks with same timestamp to verify that importwallet rescan + // will pick up both blocks, not just the first. + const int64_t BLOCK_TIME = chainActive.Tip()->GetBlockTimeMax() + 5; + SetMockTime(BLOCK_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Set key birthday to block time increased by the timestamp window, so + // rescan will start at the block time. + const int64_t KEY_TIME = BLOCK_TIME + TIMESTAMP_WINDOW; + SetMockTime(KEY_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Import key into wallet and call dumpwallet to create backup file. + { + CWallet wallet; + LOCK(wallet.cs_wallet); + wallet.mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::dumpwallet(request); + } + + // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME + // were scanned, and no prior blocks were scanned. + { + CWallet wallet; + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::importwallet(request); + + BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); + BOOST_CHECK_EQUAL(coinbaseTxns.size(), 103); + for (size_t i = 0; i < coinbaseTxns.size(); ++i) { + bool found = wallet.GetWalletTx(coinbaseTxns[i].GetHash()); + bool expected = i >= 100; + BOOST_CHECK_EQUAL(found, expected); + } + } + + SetMockTime(0); + ::pwalletMain = pwalletMainBackup; +} + // Check that GetImmatureCredit() returns a newly calculated value instead of // the cached value after a MarkDirty() call. // @@ -515,4 +578,104 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) SetMockTime(0); } +BOOST_AUTO_TEST_CASE(LoadReceiveRequests) +{ + CTxDestination dest = CKeyID(); + pwalletMain->AddDestData(dest, "misc", "val_misc"); + pwalletMain->AddDestData(dest, "rr0", "val_rr0"); + pwalletMain->AddDestData(dest, "rr1", "val_rr1"); + + auto values = pwalletMain->GetDestValues("rr"); + BOOST_CHECK_EQUAL(values.size(), 2); + BOOST_CHECK_EQUAL(values[0], "val_rr0"); + BOOST_CHECK_EQUAL(values[1], "val_rr1"); +} + +class ListCoinsTestingSetup : public TestChain100Setup +{ +public: + ListCoinsTestingSetup() + { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + ::bitdb.MakeMock(); + wallet.reset(new CWallet(std::unique_ptr<CWalletDBWrapper>(new CWalletDBWrapper(&bitdb, "wallet_test.dat")))); + bool firstRun; + wallet->LoadWallet(firstRun); + LOCK(wallet->cs_wallet); + wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + wallet->ScanForWalletTransactions(chainActive.Genesis()); + } + + ~ListCoinsTestingSetup() + { + wallet.reset(); + ::bitdb.Flush(true); + ::bitdb.Reset(); + } + + CWalletTx& AddTx(CRecipient recipient) + { + CWalletTx wtx; + CReserveKey reservekey(wallet.get()); + CAmount fee; + int changePos = -1; + std::string error; + BOOST_CHECK(wallet->CreateTransaction({recipient}, wtx, reservekey, fee, changePos, error)); + CValidationState state; + BOOST_CHECK(wallet->CommitTransaction(wtx, reservekey, nullptr, state)); + auto it = wallet->mapWallet.find(wtx.GetHash()); + BOOST_CHECK(it != wallet->mapWallet.end()); + CreateAndProcessBlock({CMutableTransaction(*it->second.tx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + it->second.SetMerkleBranch(chainActive.Tip(), 1); + return it->second; + } + + std::unique_ptr<CWallet> wallet; +}; + +BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) +{ + std::string coinbaseAddress = coinbaseKey.GetPubKey().GetID().ToString(); + LOCK(wallet->cs_wallet); + + // Confirm ListCoins initially returns 1 coin grouped under coinbaseKey + // address. + auto list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 1); + + // Check initial balance from one mature coinbase transaction. + BOOST_CHECK_EQUAL(50 * COIN, wallet->GetAvailableBalance()); + + // Add a transaction creating a change address, and confirm ListCoins still + // returns the coin associated with the change address underneath the + // coinbaseKey pubkey, even though the change address has a different + // pubkey. + AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); + + // Lock both coins. Confirm number of available coins drops to 0. + std::vector<COutput> available; + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 2); + for (const auto& group : list) { + for (const auto& coin : group.second) { + wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); + } + } + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 0); + + // Confirm ListCoins still returns same result as before, despite coins + // being locked. + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index a8f818a494..bc98435249 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -621,12 +621,9 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // if we are using HD, replace the HD master key (seed) with a new one if (IsHDEnabled()) { - CKey key; - CPubKey masterPubKey = GenerateNewHDMasterKey(); - // preserve the old chains version to not break backward compatibility - CHDChain oldChain = GetHDChain(); - if (!SetHDMasterKey(masterPubKey, &oldChain)) + if (!SetHDMasterKey(GenerateNewHDMasterKey())) { return false; + } } NewKeyPool(); @@ -985,6 +982,13 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI return false; } +bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const +{ + LOCK2(cs_main, cs_wallet); + const CWalletTx* wtx = GetWalletTx(hashTx); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() <= 0 && !wtx->InMempool(); +} + bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); @@ -1324,17 +1328,14 @@ CPubKey CWallet::GenerateNewHDMasterKey() return pubkey; } -bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) +bool CWallet::SetHDMasterKey(const CPubKey& pubkey) { LOCK(cs_wallet); // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object CHDChain newHdChain; - if (possibleOldChain) { - // preserve the old chains version - newHdChain.nVersion = possibleOldChain->nVersion; - } + newHdChain.nVersion = CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; newHdChain.masterKeyID = pubkey.GetID(); SetHDChain(newHdChain, false); @@ -1787,8 +1788,8 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { CMutableTransaction tx1 = *this->tx; CMutableTransaction tx2 = *_tx.tx; - for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); - for (unsigned int i = 0; i < tx2.vin.size(); i++) tx2.vin[i].scriptSig = CScript(); + for (auto& txin : tx1.vin) txin.scriptSig = CScript(); + for (auto& txin : tx2.vin) txin.scriptSig = CScript(); return CTransaction(tx1) == CTransaction(tx2); } @@ -1983,12 +1984,30 @@ CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth, cons return balance; } -void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl *coinControl, bool fIncludeZeroValue) const +CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + std::vector<COutput> vCoins; + AvailableCoins(vCoins, true, coinControl); + for (const COutput& out : vCoins) { + if (out.fSpendable) { + balance += out.tx->tx->vout[out.i].nValue; + } + } + return balance; +} + +void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const { vCoins.clear(); { LOCK2(cs_main, cs_wallet); + + CAmount nTotal = 0; + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const uint256& wtxid = it->first; @@ -2046,18 +2065,112 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const continue; } + if (nDepth < nMinDepth || nDepth > nMaxDepth) + continue; + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + continue; + + if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint((*it).first, i))) + continue; + + if (IsLockedCoin((*it).first, i)) + continue; + + if (IsSpent(wtxid, i)) + continue; + isminetype mine = IsMine(pcoin->tx->vout[i]); - if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue) && - (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) - vCoins.push_back(COutput(pcoin, i, nDepth, - ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || - (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO), - (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO, safeTx)); + + if (mine == ISMINE_NO) { + continue; + } + + bool fSpendableIn = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO); + bool fSolvableIn = (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO; + + vCoins.push_back(COutput(pcoin, i, nDepth, fSpendableIn, fSolvableIn, safeTx)); + + // Checks the sum amount of all UTXO's. + if (nMinimumSumAmount != MAX_MONEY) { + nTotal += pcoin->tx->vout[i].nValue; + + if (nTotal >= nMinimumSumAmount) { + return; + } + } + + // Checks the maximum number of UTXO's. + if (nMaximumCount > 0 && vCoins.size() >= nMaximumCount) { + return; + } + } + } + } +} + +std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const +{ + // TODO: Add AssertLockHeld(cs_wallet) here. + // + // Because the return value from this function contains pointers to + // CWalletTx objects, callers to this function really should acquire the + // cs_wallet lock before calling it. However, the current caller doesn't + // acquire this lock yet. There was an attempt to add the missing lock in + // https://github.com/bitcoin/bitcoin/pull/10340, but that change has been + // postponed until after https://github.com/bitcoin/bitcoin/pull/10244 to + // avoid adding some extra complexity to the Qt code. + + std::map<CTxDestination, std::vector<COutput>> result; + + std::vector<COutput> availableCoins; + AvailableCoins(availableCoins); + + LOCK2(cs_main, cs_wallet); + for (auto& coin : availableCoins) { + CTxDestination address; + if (coin.fSpendable && + ExtractDestination(FindNonChangeParentOutput(*coin.tx->tx, coin.i).scriptPubKey, address)) { + result[address].emplace_back(std::move(coin)); + } + } + + std::vector<COutPoint> lockedCoins; + ListLockedCoins(lockedCoins); + for (const auto& output : lockedCoins) { + auto it = mapWallet.find(output.hash); + if (it != mapWallet.end()) { + int depth = it->second.GetDepthInMainChain(); + if (depth >= 0 && output.n < it->second.tx->vout.size() && + IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { + CTxDestination address; + if (ExtractDestination(FindNonChangeParentOutput(*it->second.tx, output.n).scriptPubKey, address)) { + result[address].emplace_back( + &it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + } } } } + + return result; +} + +const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const +{ + const CTransaction* ptx = &tx; + int n = output; + while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) { + const COutPoint& prevout = ptx->vin[0].prevout; + auto it = mapWallet.find(prevout.hash); + if (it == mapWallet.end() || it->second.tx->vout.size() <= prevout.n || + !IsMine(it->second.tx->vout[prevout.n])) { + break; + } + ptx = it->second.tx.get(); + n = prevout.n; + } + return ptx->vout[n]; } static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, @@ -2155,10 +2268,10 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin if (nTotalLower == nTargetValue) { - for (unsigned int i = 0; i < vValue.size(); ++i) + for (const auto& input : vValue) { - setCoinsRet.insert(vValue[i]); - nValueRet += vValue[i].txout.nValue; + setCoinsRet.insert(input); + nValueRet += input.txout.nValue; } return true; } @@ -2283,10 +2396,12 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm bool CWallet::SignTransaction(CMutableTransaction &tx) { + AssertLockHeld(cs_wallet); // mapWallet + // sign the new tx CTransaction txNewConst(tx); int nIn = 0; - for (auto& input : tx.vin) { + for (const auto& input : tx.vin) { std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(input.prevout.hash); if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { return false; @@ -2455,7 +2570,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } } - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2520,16 +2635,16 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // We do not move dust-change to fees, because the sender would end up paying more than requested. // This would be against the purpose of the all-inclusive feature. // So instead we raise the change and deduct from the recipient. - if (nSubtractFeeFromAmount > 0 && newTxOut.IsDust(dustRelayFee)) + if (nSubtractFeeFromAmount > 0 && IsDust(newTxOut, ::dustRelayFee)) { - CAmount nDust = newTxOut.GetDustThreshold(dustRelayFee) - newTxOut.nValue; + CAmount nDust = GetDustThreshold(newTxOut, ::dustRelayFee) - newTxOut.nValue; newTxOut.nValue += nDust; // raise change until no more dust for (unsigned int i = 0; i < vecSend.size(); i++) // subtract from first recipient { if (vecSend[i].fSubtractFeeFromAmount) { txNew.vout[i].nValue -= nDust; - if (txNew.vout[i].IsDust(dustRelayFee)) + if (IsDust(txNew.vout[i], ::dustRelayFee)) { strFailReason = _("The transaction amount is too small to send after the fee has been deducted"); return false; @@ -2541,7 +2656,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // Never create dust outputs; if we would, just // add the dust to the fee. - if (newTxOut.IsDust(dustRelayFee)) + if (IsDust(newTxOut, ::dustRelayFee)) { nChangePosInOut = -1; nFeeRet += nChange; @@ -2605,9 +2720,6 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT currentConfirmationTarget = coinControl->nConfirmTarget; CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); - if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { - nFeeNeeded = coinControl->nMinimumTotalFee; - } if (coinControl && coinControl->fOverrideFeeRate) nFeeNeeded = coinControl->nFeeRate.GetFee(nBytes); @@ -3228,11 +3340,11 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) - if (IsMine(pcoin->tx->vout[i])) + for (const auto& txout : pcoin->tx->vout) + if (IsMine(txout)) { CTxDestination address; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, address)) + if(!ExtractDestination(txout.scriptPubKey, address)) continue; grouping.insert(address); groupings.insert(grouping); @@ -3380,7 +3492,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const return (setLockedCoins.count(outpt) > 0); } -void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) +void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { AssertLockHeld(cs_wallet); // setLockedCoins for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); @@ -3581,6 +3693,20 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st return false; } +std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const +{ + LOCK(cs_wallet); + std::vector<std::string> values; + for (const auto& address : mapAddressBook) { + for (const auto& data : address.second.destdata) { + if (!data.first.compare(0, prefix.size(), prefix)) { + values.emplace_back(data.second); + } + } + } + return values; +} + std::string CWallet::GetWalletHelpString(bool showDebug) { std::string strUsage = HelpMessageGroup(_("Wallet options:")); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 8015cc8492..11b2f7a663 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_WALLET_H #include "amount.h" +#include "policy/feerate.h" #include "streams.h" #include "tinyformat.h" #include "ui_interface.h" @@ -817,7 +818,17 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, bool fIncludeZeroValue=false) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; + + /** + * Return list of available coins and locked coins grouped by non-change output address. + */ + std::map<CTxDestination, std::vector<COutput>> ListCoins() const; + + /** + * Find non-change parent output. + */ + const CTxOut& FindNonChangeParentOutput(const CTransaction& tx, int output) const; /** * Shuffle and select coins until nTargetValue is reached while avoiding @@ -833,7 +844,7 @@ public: void LockCoin(const COutPoint& output); void UnlockCoin(const COutPoint& output); void UnlockAllCoins(); - void ListLockedCoins(std::vector<COutPoint>& vOutpts); + void ListLockedCoins(std::vector<COutPoint>& vOutpts) const; /* * Rescan abort properties @@ -872,6 +883,8 @@ public: bool LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value); //! Look up a destination data tuple in the store, return true if found false otherwise bool GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const; + //! Get all destination values matching a prefix. + std::vector<std::string> GetDestValues(const std::string& prefix) const; //! Adds a watch-only address to the store, and saves it to disk. bool AddWatchOnly(const CScript& dest, int64_t nCreateTime); @@ -916,6 +929,7 @@ public: CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; CAmount GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const; + CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; /** * Insert additional inputs into the transaction by @@ -1065,6 +1079,9 @@ public: /** Set whether this wallet broadcasts transactions. */ void SetBroadcastTransactions(bool broadcast) { fBroadcastTransactions = broadcast; } + /** Return whether transaction can be abandoned */ + bool TransactionCanBeAbandoned(const uint256& hashTx) const; + /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ bool AbandonTransaction(const uint256& hashTx); @@ -1100,9 +1117,10 @@ public: CPubKey GenerateNewHDMasterKey(); /* Set the current HD master key (will reset the chain child index counters) - If possibleOldChain is provided, the parameters from the old chain (version) - will be preserved. */ - bool SetHDMasterKey(const CPubKey& key, CHDChain *possibleOldChain = nullptr); + Sets the master key's version based on the current wallet version (so the + caller must ensure the current wallet version is correct before calling + this function). */ + bool SetHDMasterKey(const CPubKey& key); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index a90fa6dbbd..342c797dd3 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -6,9 +6,9 @@ #include "wallet/walletdb.h" #include "base58.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "fs.h" -#include "validation.h" // For CheckTransaction #include "protocol.h" #include "serialize.h" #include "sync.h" |