diff options
129 files changed, 3099 insertions, 1854 deletions
diff --git a/.travis.yml b/.travis.yml index 97bb475e4b..fcf4b082f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ env: # bitcoind - HOST=x86_64-unknown-linux-gnu PACKAGES="bc python3-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" # No wallet - - HOST=x86_64-unknown-linux-gnu PACKAGES="python3 xvfb" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=x86_64-unknown-linux-gnu PACKAGES="python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" # Cross-Mac - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy" @@ -50,8 +50,6 @@ before_script: - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS - # Start xvfb if needed, as documented at https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI - - if [ "$RUN_TESTS" = "true" -a "${DEP_OPTS#*NO_QT=1}" = "$DEP_OPTS" ]; then export DISPLAY=:99.0; /sbin/start-stop-daemon --start --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac; fi script: - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi @@ -69,7 +67,7 @@ script: - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS check VERBOSE=1; fi + - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude pruning"; fi - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --coverage --quiet ${extended}; fi after_script: diff --git a/configure.ac b/configure.ac index 3672700488..ca66216554 100644 --- a/configure.ac +++ b/configure.ac @@ -235,6 +235,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wself-assign],[CXXFLAGS="$CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[CXXFLAGS="$CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[CXXFLAGS="$CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[CXXFLAGS="$CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) fi CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 8fce648fc2..e9816f7d19 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -301,8 +301,7 @@ def main(): subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) break except subprocess.CalledProcessError as e: - print("Error signing, exiting.",file=stderr) - exit(1) + print("Error while signing, asking again.",file=stderr) elif reply == 'x': print("Not signing off on merge, exiting.",file=stderr) exit(1) diff --git a/contrib/init/bitcoind.openrcconf b/contrib/init/bitcoind.openrcconf index 0cbff6d30d..f70e25cb5f 100644 --- a/contrib/init/bitcoind.openrcconf +++ b/contrib/init/bitcoind.openrcconf @@ -23,7 +23,7 @@ #BITCOIND_NICE=0 # Additional options (avoid -conf and -datadir, use flags above) -BITCOIND_OPTS="-disablewallet" +#BITCOIND_OPTS="" # The timeout in seconds OpenRC will wait for bitcoind to terminate # after a SIGTERM has been raised. diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index afe902fd7f..139c03181f 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -8,7 +8,7 @@ and remove old versions as necessary. The seeds compiled into the release are created from sipa's DNS seed data, like this: - curl -s http://bitcoin.sipa.be/seeds.txt > seeds_main.txt + curl -s http://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt python3 makeseeds.py < seeds_main.txt > nodes_main.txt python3 generate-seeds.py . > ../../src/chainparamsseeds.h diff --git a/doc/developer-notes.md b/doc/developer-notes.md index cf860a1bf2..ec6abda91e 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -3,41 +3,64 @@ Developer Notes Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to -a single style, so please use it in new code. Old code will be converted -gradually and you are encouraged to use the provided -[clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) -to clean up the patch automatically before submitting a pull request. +a single style, which is specified below. When writing patches, favor the new +style over attempting to mimick the surrounding style, except for move-only +commits. + +Do not submit patches solely to modify the style of existing code. -- Basic rules specified in [src/.clang-format](/src/.clang-format). +- **Indentation and whitespace rules** as specified in +[src/.clang-format](/src/.clang-format). You can use the provided +[clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) +tool to clean up patches automatically before submission. - Braces on new lines for namespaces, classes, functions, methods. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parenthesis; don't do ( this ) - No space after function names; one space after `if`, `for` and `while`. - - If an `if` only has a single-statement then-clause, it can appear - on the same line as the if, without braces. In every other case, - braces are required, and the then and else clauses must appear + - If an `if` only has a single-statement `then`-clause, it can appear + on the same line as the `if`, without braces. In every other case, + braces are required, and the `then` and `else` clauses must appear correctly indented on a new line. + +- **Symbol naming conventions**. These are preferred in new code, but are not +required when doing so would need changes to significant pieces of existing +code. + - Variable and namespace names are all lowercase, and may use `_` to + separate words. + - Class member variables have a `m_` prefix. + - Global variables have a `g_` prefix. + - Constant names are all uppercase, and use `_` to separate words. + - Class names, function names and method names are CamelCase. Do not prefix + class names with `C`. + +- **Miscellaneous** - `++i` is preferred over `i++`. Block style example: ```c++ +int g_count = 0; + namespace foo { class Class { + std::string m_name; + +public: bool Function(const std::string& s, int n) { // Comment summarising what this section of code does for (int i = 0; i < n; ++i) { + int total_sum = 0; // When something fails, return early if (!Something()) return false; ... - if (SomethingElse()) { - DoMore(); + if (SomethingElse(i)) { + total_sum += ComputeSomething(g_count); } else { - DoLess(); + DoSomething(m_name, total_sum); } } diff --git a/doc/fuzzing.md b/doc/fuzzing.md index bf3ad17861..5dedcb51c8 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -32,6 +32,13 @@ We disable ccache because we don't want to pollute the ccache with instrumented objects, and similarly don't want to use non-instrumented cached objects linked in. +The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and +`afl-clang-fast++` in place of `afl-gcc` and `afl-g++` when compiling. When +compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting +`test_bitcoin_fuzzy` binary will be instrumented in such a way that the AFL +features "persistent mode" and "deferred forkserver" can be used. See +https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. + Preparing fuzzing ------------------ @@ -63,4 +70,3 @@ $AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/test_bitcoin_fuzzy You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. - diff --git a/doc/release-notes.md b/doc/release-notes.md index af792118d6..a13ede2dd5 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -36,6 +36,21 @@ Notable changes Low-level RPC changes --------------------- +- The new database model no longer stores information about transaction + versions of unspent outputs. This means that: + - The `gettxout` RPC no longer has a `version` field in the response. + - The `gettxoutsetinfo` RPC reports `hash_serialized_2` instead of `hash_serialized`, + which does not commit to the transaction versions of unspent outputs, but does + commit to the height and coinbase information. + - The `gettxoutsetinfo` response now contains `disk_size` and `bogosize` instead of + `bytes_serialized`. The first is a more accurate estimate of actual disk usage, but + is not deterministic. The second is unrelated to disk usage, but is a + database-independent metric of UTXO set size: it counts every UTXO entry as 50 + the + length of its scriptPubKey. + - The `getutxos` REST path no longer reports the `txvers` field in JSON format, + and always reports 0 for transaction versions in the binary format + + - Error codes have been updated to be more accurate for the following error cases: - `getblock` now returns RPC_MISC_ERROR if the block can't be found on disk (for example if the block has been pruned). Previously returned RPC_INTERNAL_ERROR. diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 948e13a9e1..391b9ebdf6 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -46,7 +46,8 @@ qt_test_test_bitcoin_qt_SOURCES = \ if ENABLE_WALLET qt_test_test_bitcoin_qt_SOURCES += \ qt/test/paymentservertests.cpp \ - qt/test/wallettests.cpp + qt/test/wallettests.cpp \ + wallet/test/wallet_test_fixture.cpp endif nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1e8e3d462f..5aab3381fd 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50 * CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21 * CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22 * CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 45738b5df8..cf280f485c 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -556,24 +556,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw std::runtime_error(err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]); + newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and private keys given, @@ -595,13 +597,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (!coins || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { fComplete = false; continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 31680a8ec7..5922e45801 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -117,17 +117,14 @@ bool AppInit(int argc, char* argv[]) return false; } - // Command-line RPC - bool fCommandLine = false; - for (int i = 1; i < argc; i++) - if (!IsSwitchChar(argv[i][0]) && !boost::algorithm::istarts_with(argv[i], "bitcoin:")) - fCommandLine = true; - - if (fCommandLine) - { - fprintf(stderr, "Error: There is no RPC client functionality in bitcoind anymore. Use the bitcoin-cli utility instead.\n"); - exit(EXIT_FAILURE); + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + fprintf(stderr, "Error: Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]); + exit(EXIT_FAILURE); + } } + // -server defaults to true for bitcoind but not for the GUI so do this here SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 5055fb3e0a..04a2f680fa 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -55,7 +55,7 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits return CreateGenesisBlock(pszTimestamp, genesisOutputScript, nTime, nNonce, nBits, nVersion, genesisReward); } -void CChainParams::UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +void CChainParams::UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) { consensus.vDeployments[d].nStartTime = nStartTime; consensus.vDeployments[d].nTimeout = nTimeout; @@ -356,8 +356,7 @@ void SelectParams(const std::string& network) globalChainParams = CreateChainParams(network); } -void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +void UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) { - globalChainParams->UpdateBIP9Parameters(d, nStartTime, nTimeout); + globalChainParams->UpdateVersionBitsParameters(d, nStartTime, nTimeout); } - diff --git a/src/chainparams.h b/src/chainparams.h index e5312d1080..a2f136171b 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -76,7 +76,7 @@ public: const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } const ChainTxData& TxData() const { return chainTxData; } - void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); + void UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); protected: CChainParams() {} @@ -116,8 +116,8 @@ const CChainParams &Params(); void SelectParams(const std::string& chain); /** - * Allows modifying the BIP9 regtest parameters. + * Allows modifying the Version Bits regtest parameters. */ -void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); +void UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/coins.cpp b/src/coins.cpp index b2e33abf33..5b7c562678 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,174 +4,126 @@ #include "coins.h" +#include "consensus/consensus.h" #include "memusage.h" #include "random.h" #include <assert.h> -/** - * calculate number of bytes for the bitmask, and its number of non-zero bytes - * each bit in the bitmask represents the availability of one output, but the - * availabilities of the first two outputs are encoded separately - */ -void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const { - unsigned int nLastUsedByte = 0; - for (unsigned int b = 0; 2+b*8 < vout.size(); b++) { - bool fZero = true; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) { - if (!vout[2+b*8+i].IsNull()) { - fZero = false; - continue; - } - } - if (!fZero) { - nLastUsedByte = b + 1; - nNonzeroBytes++; - } - } - nBytes += nLastUsedByte; -} - -bool CCoins::Spend(uint32_t nPos) -{ - if (nPos >= vout.size() || vout[nPos].IsNull()) - return false; - vout[nPos].SetNull(); - Cleanup(); - return true; -} - -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } +size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } - -CCoinsViewCache::~CCoinsViewCache() -{ - assert(!hasModifier); -} +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; - CCoins tmp; - if (!base->GetCoins(txid, tmp)) + Coin tmp; + if (!base->GetCoin(outpoint, tmp)) return cacheCoins.end(); - CCoinsMap::iterator ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); - if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our + CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; + if (ret->second.coin.IsSpent()) { + // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it != cacheCoins.end()) { - coins = it->second.coins; + coin = it->second.coin; return true; } return false; } -CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - size_t cachedCoinUsage = 0; - if (ret.second) { - if (!base->GetCoins(txid, ret.first->second.coins)) { - // The parent view does not have this entry; mark it as fresh. - ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } else if (ret.first->second.coins.IsPruned()) { - // The parent view only has a pruned entry for this; mark it as fresh. - ret.first->second.flags = CCoinsCacheEntry::FRESH; +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { + assert(!coin.IsSpent()); + if (coin.out.scriptPubKey.IsUnspendable()) return; + CCoinsMap::iterator it; + bool inserted; + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); + bool fresh = false; + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } + if (!possible_overwrite) { + if (!it->second.coin.IsSpent()) { + throw std::logic_error("Adding new coin that replaces non-pruned entry"); } - } else { - cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); + } + it->second.coin = std::move(coin); + it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); +} + +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { + bool fCoinbase = tx.IsCoinBase(); + const uint256& txid = tx.GetHash(); + for (size_t i = 0; i < tx.vout.size(); ++i) { + // Pass fCoinbase as the possible_overwrite flag to AddCoin, in order to correctly + // deal with the pre-BIP30 occurrances of duplicate coinbase transactions. + cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), fCoinbase); } - // Assume that whenever ModifyCoins is called, the entry will be modified. - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, cachedCoinUsage); } -/* ModifyNewCoins allows for faster coin modification when creating the new - * outputs from a transaction. It assumes that BIP 30 (no duplicate txids) - * applies and has already been tested for (or the test is not required due to - * BIP 34, height in coinbase). If we can assume BIP 30 then we know that any - * non-coinbase transaction we are adding to the UTXO must not already exist in - * the utxo unless it is fully spent. Thus we can check only if it exists DIRTY - * at the current level of the cache, in which case it is not safe to mark it - * FRESH (b/c then its spentness still needs to flushed). If it's not dirty and - * doesn't exist or is pruned in the current cache, we know it either doesn't - * exist or is pruned in parent caches, which is the definition of FRESH. The - * exception to this is the two historical violations of BIP 30 in the chain, - * both of which were coinbases. We do not mark these fresh so we we can ensure - * that they will still be properly overwritten when spent. - */ -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - if (!coinbase) { - // New coins must not already exist. - if (!ret.first->second.coins.IsPruned()) - throw std::logic_error("ModifyNewCoins should not find pre-existing coins on a non-coinbase unless they are pruned!"); - - if (!(ret.first->second.flags & CCoinsCacheEntry::DIRTY)) { - // If the coin is known to be pruned (have no unspent outputs) in - // the current view and the cache entry is not dirty, we know the - // coin also must be pruned in the parent view as well, so it is safe - // to mark this fresh. - ret.first->second.flags |= CCoinsCacheEntry::FRESH; - } +void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) return; + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + if (moveout) { + *moveout = std::move(it->second.coin); + } + if (it->second.flags & CCoinsCacheEntry::FRESH) { + cacheCoins.erase(it); + } else { + it->second.flags |= CCoinsCacheEntry::DIRTY; + it->second.coin.Clear(); } - ret.first->second.coins.Clear(); - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, 0); } -const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +static const Coin coinEmpty; + +const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { - return NULL; + return coinEmpty; } else { - return &it->second.coins; + return it->second.coin; } } -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } -bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { - CCoinsMap::const_iterator it = cacheCoins.find(txid); +bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -186,19 +138,18 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coins.IsPruned())) { + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache @@ -211,21 +162,21 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coins.IsPruned()) + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -249,11 +200,11 @@ bool CCoinsViewCache::Flush() { return fOk; } -void CCoinsViewCache::Uncache(const uint256& hash) +void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -262,13 +213,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const -{ - const CCoins* coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; -} - CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const { if (tx.IsCoinBase()) @@ -276,7 +220,7 @@ CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const CAmount nResult = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += GetOutputFor(tx.vin[i]).nValue; + nResult += AccessCoin(tx.vin[i].prevout).out.nValue; return nResult; } @@ -285,9 +229,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -295,25 +237,15 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { - assert(!cache.hasModifier); - cache.hasModifier = true; -} +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. -CCoinsModifier::~CCoinsModifier() +const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { - assert(cache.hasModifier); - cache.hasModifier = false; - it->second.coins.Cleanup(); - cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage - if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { - cache.cacheCoins.erase(it); - } else { - // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + COutPoint iter(txid, 0); + while (iter.n < MAX_OUTPUTS_PER_BLOCK) { + const Coin& alternate = view.AccessCoin(iter); + if (!alternate.IsSpent()) return alternate; + ++iter.n; } -} - -CCoinsViewCursor::~CCoinsViewCursor() -{ + return coinEmpty; } diff --git a/src/coins.h b/src/coins.h index 065bae56e9..476db8f37c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -20,135 +20,37 @@ #include <boost/foreach.hpp> #include <unordered_map> -/** - * Pruned version of CTransaction: only retains metadata and unspent transaction outputs +/** + * A UTXO entry. * * Serialized format: - * - VARINT(nVersion) - * - VARINT(nCode) - * - unspentness bitvector, for vout[2] and further; least significant byte first - * - the non-spent CTxOuts (via CTxOutCompressor) - * - VARINT(nHeight) - * - * The nCode value consists of: - * - bit 0: IsCoinBase() - * - bit 1: vout[0] is not spent - * - bit 2: vout[1] is not spent - * - The higher bits encode N, the number of non-zero bytes in the following bitvector. - * - In case both bit 1 and bit 2 are unset, they encode N-1, as there must be at - * least one non-spent output). - * - * Example: 0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e - * <><><--------------------------------------------><----> - * | \ | / - * version code vout[1] height - * - * - version = 1 - * - code = 4 (vout[1] is not spent, and 0 non-zero bytes of bitvector follow) - * - unspentness bitvector: as 0 non-zero bytes follow, it has length 0 - * - vout[1]: 835800816115944e077fe7c803cfa57f29b36bf87c1d35 - * * 8358: compact amount representation for 60000000000 (600 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 816115944e077fe7c803cfa57f29b36bf87c1d35: address uint160 - * - height = 203998 - * - * - * Example: 0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b - * <><><--><--------------------------------------------------><----------------------------------------------><----> - * / \ \ | | / - * version code unspentness vout[4] vout[16] height - * - * - version = 1 - * - code = 9 (coinbase, neither vout[0] or vout[1] are unspent, - * 2 (1, +1 because both bit 1 and bit 2 are unset) non-zero bitvector bytes follow) - * - unspentness bitvector: bits 2 (0x04) and 14 (0x4000) are set, so vout[2+2] and vout[14+2] are unspent - * - vout[4]: 86ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4ee - * * 86ef97d579: compact amount representation for 234925952 (2.35 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 61b01caab50f1b8e9c50a5057eb43c2d9563a4ee: address uint160 - * - vout[16]: bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4 - * * bbd123: compact amount representation for 110397 (0.001 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 8c988f1a4a4de2161e0f50aac7f17e7f9555caa4: address uint160 - * - height = 120891 + * - VARINT((coinbase ? 1 : 0) | (height << 1)) + * - the non-spent CTxOut (via CTxOutCompressor) */ -class CCoins +class Coin { public: - //! whether transaction is a coinbase - bool fCoinBase; - - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector<CTxOut> vout; + //! unspent transaction output + CTxOut out; - //! at which height this transaction was included in the active block chain - int nHeight; + //! whether containing transaction was a coinbase + unsigned int fCoinBase : 1; - //! version of the CTransaction; accesses to this value should probably check for nHeight as well, - //! as new tx version will probably only be introduced at certain heights - int nVersion; + //! at which height this containing transaction was included in the active block chain + uint32_t nHeight : 31; - void FromTx(const CTransaction &tx, int nHeightIn) { - fCoinBase = tx.IsCoinBase(); - vout = tx.vout; - nHeight = nHeightIn; - nVersion = tx.nVersion; - ClearUnspendable(); - } - - //! construct a CCoins from a CTransaction, at a given height - CCoins(const CTransaction &tx, int nHeightIn) { - FromTx(tx, nHeightIn); - } + //! construct a Coin from a CTxOut and height/coinbase information. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn),nHeight(nHeightIn) {} void Clear() { + out.SetNull(); fCoinBase = false; - std::vector<CTxOut>().swap(vout); nHeight = 0; - nVersion = 0; } //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0), nVersion(0) { } - - //!remove spent outputs at the end of vout - void Cleanup() { - while (vout.size() > 0 && vout.back().IsNull()) - vout.pop_back(); - if (vout.empty()) - std::vector<CTxOut>().swap(vout); - } - - void ClearUnspendable() { - BOOST_FOREACH(CTxOut &txout, vout) { - if (txout.scriptPubKey.IsUnspendable()) - txout.SetNull(); - } - Cleanup(); - } - - void swap(CCoins &to) { - std::swap(to.fCoinBase, fCoinBase); - to.vout.swap(vout); - std::swap(to.nHeight, nHeight); - std::swap(to.nVersion, nVersion); - } - - //! equality test - friend bool operator==(const CCoins &a, const CCoins &b) { - // Empty CCoins objects are always equal. - if (a.IsPruned() && b.IsPruned()) - return true; - return a.fCoinBase == b.fCoinBase && - a.nHeight == b.nHeight && - a.nVersion == b.nVersion && - a.vout == b.vout; - } - friend bool operator!=(const CCoins &a, const CCoins &b) { - return !(a == b); - } - - void CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const; + Coin() : fCoinBase(false), nHeight(0) { } bool IsCoinBase() const { return fCoinBase; @@ -156,115 +58,52 @@ public: template<typename Stream> void Serialize(Stream &s) const { - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - ::Serialize(s, VARINT(this->nVersion)); - // header code - ::Serialize(s, VARINT(nCode)); - // spentness bitmask - for (unsigned int b = 0; b<nMaskSize; b++) { - unsigned char chAvail = 0; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) - if (!vout[2+b*8+i].IsNull()) - chAvail |= (1 << i); - ::Serialize(s, chAvail); - } - // txouts themself - for (unsigned int i = 0; i < vout.size(); i++) { - if (!vout[i].IsNull()) - ::Serialize(s, CTxOutCompressor(REF(vout[i]))); - } - // coinbase height - ::Serialize(s, VARINT(nHeight)); + assert(!IsSpent()); + uint32_t code = nHeight * 2 + fCoinBase; + ::Serialize(s, VARINT(code)); + ::Serialize(s, CTxOutCompressor(REF(out))); } template<typename Stream> void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - ::Unserialize(s, VARINT(this->nVersion)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector<bool> vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); - } - // coinbase height - ::Unserialize(s, VARINT(nHeight)); - Cleanup(); + uint32_t code = 0; + ::Unserialize(s, VARINT(code)); + nHeight = code >> 1; + fCoinBase = code & 1; + ::Unserialize(s, REF(CTxOutCompressor(out))); } - //! mark a vout spent - bool Spend(uint32_t nPos); - - //! check whether a particular output is still available - bool IsAvailable(unsigned int nPos) const { - return (nPos < vout.size() && !vout[nPos].IsNull()); - } - - //! check whether the entire CCoins is spent - //! note that only !IsPruned() CCoins can be serialized - bool IsPruned() const { - BOOST_FOREACH(const CTxOut &out, vout) - if (!out.IsNull()) - return false; - return true; + bool IsSpent() const { + return out.IsNull(); } size_t DynamicMemoryUsage() const { - size_t ret = memusage::DynamicUsage(vout); - BOOST_FOREACH(const CTxOut &out, vout) { - ret += RecursiveDynamicUsage(out.scriptPubKey); - } - return ret; + return memusage::DynamicUsage(out.scriptPubKey); } }; -class SaltedTxidHasher +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint& id) const { + return SipHashUint256Extra(k0, k1, id.hash, id.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + Coin coin; // The actual cached data. unsigned char flags; enum Flags { @@ -277,20 +116,21 @@ struct CCoinsCacheEntry */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} }; -typedef std::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; +typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -306,17 +146,17 @@ private: class CCoinsView { public: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; + //! Retrieve the Coin (unspent transaction output) for a given outpoint. + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; + //! Just check whether we have data for a given outpoint. + //! This may (but cannot always) return true for spent outputs. + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -325,6 +165,9 @@ public: //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} + + //! Estimate database size (0 if not implemented) + virtual size_t EstimateSize() const { return 0; } }; @@ -336,45 +179,20 @@ protected: public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + size_t EstimateSize() const override; }; -class CCoinsViewCache; - -/** - * A reference to a mutable cache entry. Encapsulating it allows us to run - * cleanup code after the modification is finished, and keeping track of - * concurrent modifications. - */ -class CCoinsModifier -{ -private: - CCoinsViewCache& cache; - CCoinsMap::iterator it; - size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); - -public: - CCoins* operator->() { return &it->second.coins; } - CCoins& operator*() { return it->second.coins; } - ~CCoinsModifier(); - friend class CCoinsViewCache; -}; - /** CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: - /* Whether this cache has an active modifier. */ - bool hasModifier; - - /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -382,51 +200,45 @@ protected: mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); - ~CCoinsViewCache(); // Standard CCoinsView methods - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); /** - * Check if we have the given tx already loaded in this cache. - * The semantics are the same as HaveCoins(), but no calls to + * Check if we have the given utxo already loaded in this cache. + * The semantics are the same as HaveCoin(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const uint256 &txid) const; + bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a pointer to CCoins in the cache, or NULL if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are + * Return a reference to Coin in the cache, or a pruned one if not found. This is + * more efficient than GetCoin. Modifications to other cache entries are * allowed while accessing the returned pointer. */ - const CCoins* AccessCoins(const uint256 &txid) const; + const Coin& AccessCoin(const COutPoint &output) const; /** - * Return a modifiable reference to a CCoins. If no entry with the given - * txid exists, a new one is created. Simultaneous modifications are not - * allowed. + * Add a coin. Set potential_overwrite to true if a non-pruned version may + * already exist. */ - CCoinsModifier ModifyCoins(const uint256 &txid); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); /** - * Return a modifiable reference to a CCoins. Assumes that no entry with the given - * txid exists and creates a new one. This saves a database access in the case where - * the coins were to be wiped out by FromTx anyway. This should not be called with - * the 2 historical coinbase duplicate pairs because the new coins are marked fresh, and - * in the event the duplicate coinbase was spent before a flush, the now pruned coins - * would not properly overwrite the first coinbase of the pair. Simultaneous modifications - * are not allowed. + * Spend a coin. Pass moveto in order to get the deleted data. + * If no unspent output exists for the passed outpoint, this call + * has no effect. */ - CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); + void SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** * Push the modifications applied to this cache to its base. @@ -436,12 +248,12 @@ public: bool Flush(); /** - * Removes the transaction with the given hash from the cache, if it is + * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ - void Uncache(const uint256 &txid); + void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -460,12 +272,8 @@ public: //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - const CTxOut &GetOutputFor(const CTxIn& input) const; - - friend class CCoinsModifier; - private: - CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -473,4 +281,13 @@ private: CCoinsViewCache(const CCoinsViewCache &); }; +//! Utility function to add all of a transaction's outputs to a cache. +// It assumes that overwrites are only possible for coinbase transactions, +// TODO: pass in a boolean to limit these possible overwrites to known +// (pre-BIP34) cases. +void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); + +//! Utility function to find any unspent output with a given txid. +const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); + #endif // BITCOIN_COINS_H diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 043f4cf95c..bf68f8754b 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -126,7 +126,7 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in unsigned int nSigOps = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; if (prevout.scriptPubKey.IsPayToScriptHash()) nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); } @@ -146,7 +146,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); } return nSigOps; @@ -213,20 +213,20 @@ bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, c for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { - if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) + if (coin.IsCoinBase()) { + if (nSpendHeight - coin.nHeight < COINBASE_MATURITY) return state.Invalid(false, REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); } // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } diff --git a/src/core_memusage.h b/src/core_memusage.h index 5e10182075..e4ccd54c42 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -63,4 +63,9 @@ static inline size_t RecursiveDynamicUsage(const CBlockLocator& locator) { return memusage::DynamicUsage(locator.vHave); } +template<typename X> +static inline size_t RecursiveDynamicUsage(const std::shared_ptr<X>& p) { + return p ? memusage::DynamicUsage(p) + RecursiveDynamicUsage(*p) : 0; +} + #endif // BITCOIN_CORE_MEMUSAGE_H diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b13e98b7a4..24ef71bfbf 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -55,11 +55,19 @@ private: CDataStream ssKey; CDataStream ssValue; + size_t size_estimate; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + + void Clear() + { + batch.Clear(); + size_estimate = 0; + } template <typename K, typename V> void Write(const K& key, const V& value) @@ -74,6 +82,14 @@ public: leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } @@ -86,8 +102,16 @@ public: leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } + + size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator @@ -281,7 +305,22 @@ public: * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); + + template<typename K> + size_t EstimateSize(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; + } }; #endif // BITCOIN_DBWRAPPER_H - diff --git a/src/hash.cpp b/src/hash.cpp index a14a2386a2..b361c90d16 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -208,3 +208,44 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } + +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = (((uint64_t)36) << 56) | extra; + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index eacb8f04fe..b9952d39fc 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,6 +160,41 @@ public: } }; +/** Reads data from an underlying stream, while hashing the read data. */ +template<typename Source> +class CHashVerifier : public CHashWriter +{ +private: + Source* source; + +public: + CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} + + void read(char* pch, size_t nSize) + { + source->read(pch, nSize); + this->write(pch, nSize); + } + + void ignore(size_t nSize) + { + char data[1024]; + while (nSize > 0) { + size_t now = std::min<size_t>(nSize, 1024); + read(data, now); + nSize -= now; + } + } + + template<typename T> + CHashVerifier<Source>& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template<typename T> uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) @@ -206,5 +241,6 @@ public: * .Finalize() */ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra); #endif // BITCOIN_HASH_H diff --git a/src/init.cpp b/src/init.cpp index 3bbdb16c3b..04d1280c92 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -146,9 +146,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); @@ -449,7 +449,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); - strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified BIP9 deployment (regtest-only)"); + strUsage += HelpMessageOpt("-vbparams=deployment:start:end", "Use given start/end times for specified version bits deployment (regtest-only)"); } strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " + _("If <category> is not supplied or if <category> = 1, output all debugging information.") + " " + _("<category> can be:") + " " + ListLogCategories() + "."); @@ -1104,16 +1104,16 @@ bool AppInitParameterInteraction() fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } - if (gArgs.IsArgSet("-bip9params")) { - // Allow overriding BIP9 parameters for testing + if (gArgs.IsArgSet("-vbparams")) { + // Allow overriding version bits parameters for testing if (!chainparams.MineBlocksOnDemand()) { - return InitError("BIP9 parameters may only be overridden on regtest."); + return InitError("Version bits parameters may only be overridden on regtest."); } - for (const std::string& strDeployment : gArgs.GetArgs("-bip9params")) { + for (const std::string& strDeployment : gArgs.GetArgs("-vbparams")) { std::vector<std::string> vDeploymentParams; boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); if (vDeploymentParams.size() != 3) { - return InitError("BIP9 parameters malformed, expecting deployment:start:end"); + return InitError("Version bits parameters malformed, expecting deployment:start:end"); } int64_t nStartTime, nTimeout; if (!ParseInt64(vDeploymentParams[1], &nStartTime)) { @@ -1126,9 +1126,9 @@ bool AppInitParameterInteraction() for (int j=0; j<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[j].name) == 0) { - UpdateBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); + UpdateVersionBitsParameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); found = true; - LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); + LogPrintf("Setting version bits activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); break; } } @@ -1386,11 +1386,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (gArgs.IsArgSet("-seednode")) { - BOOST_FOREACH(const std::string& strDest, gArgs.GetArgs("-seednode")) - connman.AddOneShot(strDest); - } - #if ENABLE_ZMQ pzmqNotificationInterface = CZMQNotificationInterface::Create(); @@ -1455,6 +1450,12 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); + } else { + // If necessary, upgrade from older database format. + if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; + } } if (!LoadBlockIndex(chainparams)) { @@ -1659,6 +1660,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; + if (gArgs.IsArgSet("-seednode")) { + connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); + } + if (!connman.Start(scheduler, strNodeError, connOptions)) return InitError(strNodeError); diff --git a/src/net.cpp b/src/net.cpp index ded6f1099a..14ac5618eb 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -148,7 +148,7 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn // one by discovery. CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) { - CAddress ret(CService(CNetAddr(),GetListenPort()), NODE_NONE); + CAddress ret(CService(CNetAddr(),GetListenPort()), nLocalServices); CService addr; if (GetLocal(addr, paddrPeer)) { @@ -340,6 +340,22 @@ bool CConnman::CheckIncomingNonce(uint64_t nonce) return true; } +/** Get the bind address for a socket as CAddress */ +static CAddress GetBindAddress(SOCKET sock) +{ + CAddress addr_bind; + struct sockaddr_storage sockaddr_bind; + socklen_t sockaddr_bind_len = sizeof(sockaddr_bind); + if (sock != INVALID_SOCKET) { + if (!getsockname(sock, (struct sockaddr*)&sockaddr_bind, &sockaddr_bind_len)) { + addr_bind.SetSockAddr((const struct sockaddr*)&sockaddr_bind); + } else { + LogPrint(BCLog::NET, "Warning: getsockname failed\n"); + } + } + return addr_bind; +} + CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure) { if (pszDest == NULL) { @@ -393,7 +409,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo // Add node NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, pszDest ? pszDest : "", false); + CAddress addr_bind = GetBindAddress(hSocket); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); @@ -635,6 +652,7 @@ void CNode::copyStats(CNodeStats &stats) stats.nodeid = this->GetId(); X(nServices); X(addr); + X(addrBind); { LOCK(cs_filter); X(fRelayTxes); @@ -1036,9 +1054,11 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { int nInbound = 0; int nMaxInbound = nMaxConnections - (nMaxOutbound + nMaxFeeler); - if (hSocket != INVALID_SOCKET) - if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) + if (hSocket != INVALID_SOCKET) { + if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { LogPrintf("Warning: Unknown socket family\n"); + } + } bool whitelisted = hListenSocket.whitelisted || IsWhitelistedRange(addr); { @@ -1092,8 +1112,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, "", true); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; GetNodeSignals().InitializeNode(pnode, *this); @@ -1721,11 +1742,17 @@ void CConnman::ThreadOpenConnections() // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; + int nOutboundRelevant = 0; std::set<std::vector<unsigned char> > setConnected; { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { if (!pnode->fInbound && !pnode->fAddnode) { + + // Count the peers that have all relevant services + if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { + nOutboundRelevant++; + } // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1743,9 +1770,9 @@ void CConnman::ThreadOpenConnections() // * Increase the number of connectable addresses in the tried table. // // Method: - // * Choose a random address from new and attempt to connect to it if we can connect + // * Choose a random address from new and attempt to connect to it if we can connect // successfully it is added to tried. - // * Start attempting feeler connections only after node finishes making outbound + // * Start attempting feeler connections only after node finishes making outbound // connections. // * Only make a feeler connection once every few minutes. // @@ -1789,14 +1816,27 @@ void CConnman::ThreadOpenConnections() continue; // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - if ((addr.nServices & nRelevantServices) != nRelevantServices && (nTries < 40 || nOutbound >= (nMaxOutbound >> 1))) + ServiceFlags nRequiredServices = nRelevantServices; + if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { + nRequiredServices = REQUIRED_SERVICES; + } + + if ((addr.nServices & nRequiredServices) != nRequiredServices) { continue; + } // do not allow non-default ports, unless after 50 invalid addresses selected already if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) continue; addrConnect = addr; + + // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services + if (nOutboundRelevant >= (nMaxOutbound >> 1)) { + addrConnect.nServices = REQUIRED_SERVICES; + } else { + addrConnect.nServices = nRequiredServices; + } break; } @@ -2212,6 +2252,10 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c SetBestHeight(connOptions.nBestHeight); + for (const auto& strDest : connOptions.vSeedNodes) { + AddOneShot(strDest); + } + clientInterface = connOptions.uiInterface; if (clientInterface) { clientInterface->InitMessage(_("Loading P2P addresses...")); @@ -2616,9 +2660,10 @@ int CConnman::GetBestHeight() const unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } unsigned int CConnman::GetSendBufferSize() const{ return nSendBufferMaxSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string& addrNameIn, bool fInboundIn) : +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string& addrNameIn, bool fInboundIn) : nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), + addrBind(addrBindIn), fInbound(fInboundIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), @@ -144,6 +144,7 @@ public: unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; uint64_t nMaxOutboundLimit = 0; + std::vector<std::string> vSeedNodes; }; CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); @@ -233,8 +234,6 @@ public: void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); - void AddOneShot(const std::string& strDest); - bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); @@ -292,6 +291,7 @@ private: }; void ThreadOpenAddedConnections(); + void AddOneShot(const std::string& strDest); void ProcessOneShot(); void ThreadOpenConnections(); void ThreadMessageHandler(); @@ -504,8 +504,12 @@ public: double dPingTime; double dPingWait; double dMinPing; + // Our address, as reported by the peer std::string addrLocal; + // Address of this peer CAddress addr; + // Bind address of our side of the connection + CAddress addrBind; }; @@ -586,7 +590,10 @@ public: std::atomic<int64_t> nLastRecv; const int64_t nTimeConnected; std::atomic<int64_t> nTimeOffset; + // Address of this peer const CAddress addr; + // Bind address of our side of the connection + const CAddress addrBind; std::atomic<int> nVersion; // strSubVer is whatever byte array we read from the wire. However, this field is intended // to be printed out, displayed to humans in various forms and so on. So we sanitize it and @@ -676,7 +683,7 @@ public: CAmount lastSentFeeFilter; int64_t nextSendTimeFeeFilter; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string &addrNameIn = "", bool fInboundIn = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); private: @@ -695,6 +702,7 @@ private: mutable CCriticalSection cs_addrName; std::string addrName; + // Our address, as reported by the peer CService addrLocal; mutable CCriticalSection cs_addrLocal; public: @@ -711,7 +719,7 @@ public: return nMyStartingHeight; } - int GetRefCount() + int GetRefCount() const { assert(nRefCount >= 0); return nRefCount; @@ -723,7 +731,7 @@ public: { nRecvVersion = nVersionIn; } - int GetRecvVersion() + int GetRecvVersion() const { return nRecvVersion; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index aca88f2660..4ca02c281d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -168,6 +168,8 @@ struct CNodeState { int nUnconnectingHeaders; //! Whether we've started headers synchronization with this peer. bool fSyncStarted; + //! When to potentially disconnect peer for stalling headers download + int64_t nHeadersSyncTimeout; //! Since when we're stalling block download progress (in microseconds), or 0. int64_t nStallingSince; std::list<QueuedBlock> vBlocksInFlight; @@ -207,6 +209,7 @@ struct CNodeState { pindexBestHeaderSent = NULL; nUnconnectingHeaders = 0; fSyncStarted = false; + nHeadersSyncTimeout = 0; nStallingSince = 0; nDownloadingSince = 0; nBlocksInFlight = 0; @@ -481,7 +484,7 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<con // Make sure pindexBestKnownBlock is up to date, we'll need it. ProcessBlockAvailability(nodeid); - if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork) { + if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < UintToArith256(consensusParams.nMinimumChainWork)) { // This peer has nothing interesting. return; } @@ -754,8 +757,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb const CTransaction& tx = *ptx; // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + for (const auto& txin : tx.vin) { + auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { const CTransaction& orphanTx = *(*mi)->second.tx; @@ -911,12 +914,11 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -1553,10 +1555,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr uint32_t nFetchFlags = GetFetchFlags(pfrom); - for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) + for (CInv &inv : vInv) { - CInv &inv = vInv[nInv]; - if (interruptMsgProc) return true; @@ -2884,6 +2884,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; + state.nHeadersSyncTimeout = GetTimeMicros() + HEADERS_DOWNLOAD_TIMEOUT_BASE + HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER * (GetAdjustedTime() - pindexBestHeader->GetBlockTime())/(consensusParams.nPowTargetSpacing); nSyncStarted++; const CBlockIndex *pindexStart = pindexBestHeader; /* If possible, start at the block preceding the currently @@ -3207,6 +3208,39 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr return true; } } + // Check for headers sync timeouts + if (state.fSyncStarted && state.nHeadersSyncTimeout < std::numeric_limits<int64_t>::max()) { + // Detect whether this is a stalling initial-headers-sync peer + if (pindexBestHeader->GetBlockTime() <= GetAdjustedTime() - 24*60*60) { + if (nNow > state.nHeadersSyncTimeout && nSyncStarted == 1 && (nPreferredDownload - state.fPreferredDownload >= 1)) { + // Disconnect a (non-whitelisted) peer if it is our only sync peer, + // and we have others we could be using instead. + // Note: If all our peers are inbound, then we won't + // disconnect our sync peer for stalling; we have bigger + // problems if we can't get any outbound peers. + if (!pto->fWhitelisted) { + LogPrintf("Timeout downloading headers from peer=%d, disconnecting\n", pto->GetId()); + pto->fDisconnect = true; + return true; + } else { + LogPrintf("Timeout downloading headers from whitelisted peer=%d, not disconnecting\n", pto->GetId()); + // Reset the headers sync state so that we have a + // chance to try downloading from a different peer. + // Note: this will also result in at least one more + // getheaders message to be sent to + // this peer (eventually). + state.fSyncStarted = false; + nSyncStarted--; + state.nHeadersSyncTimeout = 0; + } + } + } else { + // After we've caught up once, reset the timeout so we can't trigger + // disconnect later. + state.nHeadersSyncTimeout = std::numeric_limits<int64_t>::max(); + } + } + // // Message: getdata (blocks) diff --git a/src/net_processing.h b/src/net_processing.h index f460595bc1..db6d81e6b6 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -17,6 +17,10 @@ static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; +/** Headers download timeout expressed in microseconds + * Timeout = base + per_header * (expected number of headers) */ +static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes +static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header /** Register with a network node to receive its signals */ void RegisterNodeSignals(CNodeSignals& nodeSignals); diff --git a/src/netbase.cpp b/src/netbase.cpp index 2584f571ee..32557dd179 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -668,13 +668,14 @@ std::string NetworkErrorString(int err) std::string NetworkErrorString(int err) { char buf[256]; - const char *s = buf; buf[0] = 0; /* Too bad there are two incompatible implementations of the * thread-safe strerror. */ + const char *s; #ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ s = strerror_r(err, buf, sizeof(buf)); #else /* POSIX variant always returns message in buffer */ + s = buf; if (strerror_r(err, buf, sizeof(buf))) buf[0] = 0; #endif diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index bad3de4b44..7a9af5edc2 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -604,8 +604,8 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, unsigned int countedTxs = 0; // Update averages with data points from current block - for (unsigned int i = 0; i < entries.size(); i++) { - if (processBlockTx(nBlockHeight, entries[i])) + for (const auto& entry : entries) { + if (processBlockTx(nBlockHeight, entry)) countedTxs++; } @@ -786,7 +786,7 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun * This is necessary to preserve monotonically increasing estimates. * For non-conservative estimates we do the same thing for 2*target, but * for conservative estimates we want to skip these shorter horizons - * checks for 2*target becuase we are taking the max over all time + * checks for 2*target because we are taking the max over all time * horizons so we already have monotonically increasing estimates and * the purpose of conservative estimates is not to let short term * fluctuations lower our estimates by too much. @@ -944,7 +944,7 @@ void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { removeTx(txid, false); } int64_t endclear = GetTimeMicros(); - LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %ld micros\n",txids.size(), endclear - startclear); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n",txids.size(), (endclear - startclear)*0.000001); } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index f527c85004..e99fec2c39 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -51,7 +51,7 @@ class TxConfirmStats; * blocks. This is is tracked in 3 different data sets each up to a maximum * number of periods. Within each data set we have an array of counters in each * feerate bucket and we increment all the counters from Y' up to max periods - * representing that a tx was successfullly confirmed in less than or equal to + * representing that a tx was successfully confirmed in less than or equal to * that many periods. We want to save a history of this information, so at any * time we have a counter of the total number of transactions that happened in a * given feerate bucket and the total number that were confirmed in each of the diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 2b19a6714b..14d58e7442 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -15,7 +15,7 @@ #include <boost/foreach.hpp> -CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee) +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) { // "Dust" is defined in terms of dustRelayFee, // which has units satoshis-per-kilobyte. @@ -44,12 +44,12 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee) nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above } - return 3 * dustRelayFee.GetFee(nSize); + return 3 * dustRelayFeeIn.GetFee(nSize); } -bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee) +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) { - return (txout.nValue < GetDustThreshold(txout, dustRelayFee)); + return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); } /** @@ -165,7 +165,7 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; txnouttype whichType; @@ -204,7 +204,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) if (tx.vin[i].scriptWitness.IsNull()) continue; - const CTxOut &prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut &prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; // get the scriptPubKey corresponding to this input: CScript prevScript = prev.scriptPubKey; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 2a331d4fae..135cf6f701 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -513,8 +513,6 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) // Fee nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); - if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) - nPayFee = coinControl->nMinimumTotalFee; if (nPayAmount > 0) { @@ -573,7 +571,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) l5->setText(((nBytes > 0) ? ASYMP_UTF8 : "") + QString::number(nBytes)); // Bytes l7->setText(fDust ? tr("yes") : tr("no")); // Dust l8->setText(BitcoinUnits::formatWithUnit(nDisplayUnit, nChange)); // Change - if (nPayFee > 0 && (coinControl->nMinimumTotalFee < nPayFee)) + if (nPayFee > 0) { l3->setText(ASYMP_UTF8 + l3->text()); l4->setText(ASYMP_UTF8 + l4->text()); diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 52256ca5c4..89f9c25d14 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -862,19 +862,6 @@ </widget> </item> <item> - <widget class="QRadioButton" name="radioCustomAtLeast"> - <property name="toolTip"> - <string>If the custom fee is set to 1000 satoshis and the transaction is only 250 bytes, then "per kilobyte" only pays 250 satoshis in fee, while "total at least" pays 1000 satoshis. For transactions bigger than a kilobyte both pay by kilobyte.</string> - </property> - <property name="text"> - <string>total at least</string> - </property> - <attribute name="buttonGroup"> - <string notr="true">groupCustomFee</string> - </attribute> - </widget> - </item> - <item> <widget class="BitcoinAmountField" name="customFee"/> </item> <item> diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 8dcb0fd016..272ab9486a 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -109,7 +109,6 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupFee->setId(ui->radioCustomFee, 1); ui->groupFee->button((int)std::max(0, std::min(1, settings.value("nFeeRadio").toInt())))->setChecked(true); ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); - ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); @@ -606,7 +605,6 @@ void SendCoinsDialog::updateFeeSectionControls() ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); - ui->radioCustomAtLeast ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() && CoinControlDialog::coinControl->HasSelected()); ui->customFee ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); } @@ -617,19 +615,12 @@ void SendCoinsDialog::updateGlobalFeeVariables() int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; payTxFee = CFeeRate(0); - // set nMinimumTotalFee to 0 to not accidentally pay a custom fee - CoinControlDialog::coinControl->nMinimumTotalFee = 0; - // show the estimated required time for confirmation ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget * Params().GetConsensus().nPowTargetSpacing) + " / " + tr("%n block(s)", "", nConfirmTarget)); } else { payTxFee = CFeeRate(ui->customFee->value()); - - // if user has selected to set a minimum absolute fee, pass the value to coincontrol - // set nMinimumTotalFee to 0 in case of user has selected that the fee is per KB - CoinControlDialog::coinControl->nMinimumTotalFee = ui->radioCustomAtLeast->isChecked() ? ui->customFee->value() : 0; } } @@ -828,21 +819,6 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - if (model->getOptionsModel()->getCoinControlFeatures()) - { - // enable minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(true); - - // only enable the feature if inputs are selected - ui->radioCustomAtLeast->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() &&CoinControlDialog::coinControl->HasSelected()); - } - else - { - // in case coin control is disabled (=default), hide minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(false); - return; - } - // set pay amounts CoinControlDialog::payAmounts.clear(); CoinControlDialog::fSubtractFeeFromAmount = false; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index a0dce3d997..ff1eb59f16 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -8,26 +8,46 @@ #include "qt/sendcoinsdialog.h" #include "qt/sendcoinsentry.h" #include "qt/transactiontablemodel.h" +#include "qt/transactionview.h" #include "qt/walletmodel.h" #include "test/test_bitcoin.h" #include "validation.h" #include "wallet/wallet.h" #include <QAbstractButton> +#include <QAction> #include <QApplication> +#include <QCheckBox> +#include <QPushButton> #include <QTimer> #include <QVBoxLayout> namespace { -//! Press "Yes" button in modal send confirmation dialog. -void ConfirmSend() +//! Press "Ok" button in message box dialog. +void ConfirmMessage(QString* text = nullptr) { - QTimer::singleShot(0, makeCallback([](Callback* callback) { + QTimer::singleShot(0, makeCallback([text](Callback* callback) { + for (QWidget* widget : QApplication::topLevelWidgets()) { + if (widget->inherits("QMessageBox")) { + QMessageBox* messageBox = qobject_cast<QMessageBox*>(widget); + if (text) *text = messageBox->text(); + messageBox->defaultButton()->click(); + } + } + delete callback; + }), SLOT(call())); +} + +//! Press "Yes" or "Cancel" buttons in modal send confirmation dialog. +void ConfirmSend(QString* text = nullptr, bool cancel = false) +{ + QTimer::singleShot(0, makeCallback([text, cancel](Callback* callback) { for (QWidget* widget : QApplication::topLevelWidgets()) { if (widget->inherits("SendConfirmationDialog")) { SendConfirmationDialog* dialog = qobject_cast<SendConfirmationDialog*>(widget); - QAbstractButton* button = dialog->button(QMessageBox::Yes); + if (text) *text = dialog->text(); + QAbstractButton* button = dialog->button(cancel ? QMessageBox::Cancel : QMessageBox::Yes); button->setEnabled(true); button->click(); } @@ -37,12 +57,16 @@ void ConfirmSend() } //! Send coins to address and return txid. -uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount) +uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount, bool rbf) { QVBoxLayout* entries = sendCoinsDialog.findChild<QVBoxLayout*>("entries"); SendCoinsEntry* entry = qobject_cast<SendCoinsEntry*>(entries->itemAt(0)->widget()); entry->findChild<QValidatedLineEdit*>("payTo")->setText(QString::fromStdString(address.ToString())); entry->findChild<BitcoinAmountField*>("payAmount")->setValue(amount); + sendCoinsDialog.findChild<QFrame*>("frameFee") + ->findChild<QFrame*>("frameFeeSelection") + ->findChild<QCheckBox*>("optInRBF") + ->setCheckState(rbf ? Qt::Checked : Qt::Unchecked); uint256 txid; boost::signals2::scoped_connection c(wallet.NotifyTransactionChanged.connect([&txid](CWallet*, const uint256& hash, ChangeType status) { if (status == CT_NEW) txid = hash; @@ -65,6 +89,42 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) } return {}; } + +//! Request context menu (call method that is public in qt5, but protected in qt4). +void RequestContextMenu(QWidget* widget) +{ + class Qt4Hack : public QWidget + { + public: + using QWidget::customContextMenuRequested; + }; + static_cast<Qt4Hack*>(widget)->customContextMenuRequested({}); +} + +//! Invoke bumpfee on txid and check results. +void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, std::string expectError, bool cancel) +{ + QTableView* table = view.findChild<QTableView*>("transactionView"); + QModelIndex index = FindTx(*table->selectionModel()->model(), txid); + QVERIFY2(index.isValid(), "Could not find BumpFee txid"); + + // Select row in table, invoke context menu, and make sure bumpfee action is + // enabled or disabled as expected. + QAction* action = view.findChild<QAction*>("bumpFeeAction"); + table->selectionModel()->select(index, QItemSelectionModel::ClearAndSelect | QItemSelectionModel::Rows); + action->setEnabled(expectDisabled); + RequestContextMenu(table); + QCOMPARE(action->isEnabled(), !expectDisabled); + + action->setEnabled(true); + QString text; + if (expectError.empty()) { + ConfirmSend(&text, cancel); + } else { + ConfirmMessage(&text); + } + action->trigger(); + QVERIFY(text.indexOf(QString::fromStdString(expectError)) != -1); } //! Simple qt wallet tests. @@ -80,11 +140,13 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) // src/qt/test/test_bitcoin-qt -platform xcb # Linux // src/qt/test/test_bitcoin-qt -platform windows # Windows // src/qt/test/test_bitcoin-qt -platform cocoa # macOS -void WalletTests::walletTests() +void TestSendCoins() { - // Set up wallet and chain with 101 blocks (1 mature block for spending). + // Set up wallet and chain with 105 blocks (5 mature blocks for spending). TestChain100Setup test; - test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + for (int i = 0; i < 5; ++i) { + test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + } bitdb.MakeMock(); std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); CWallet wallet(std::move(dbw)); @@ -101,19 +163,34 @@ void WalletTests::walletTests() // Create widgets for sending coins and listing transactions. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); + TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel; WalletModel walletModel(platformStyle.get(), &wallet, &optionsModel); sendCoinsDialog.setModel(&walletModel); + transactionView.setModel(&walletModel); // Send two transactions, and verify they are added to transaction list. TransactionTableModel* transactionTableModel = walletModel.getTransactionTableModel(); - QCOMPARE(transactionTableModel->rowCount({}), 101); - uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN); - uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN); - QCOMPARE(transactionTableModel->rowCount({}), 103); + QCOMPARE(transactionTableModel->rowCount({}), 105); + uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN, false /* rbf */); + uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN, true /* rbf */); + QCOMPARE(transactionTableModel->rowCount({}), 107); QVERIFY(FindTx(*transactionTableModel, txid1).isValid()); QVERIFY(FindTx(*transactionTableModel, txid2).isValid()); + // Call bumpfee. Test disabled, canceled, enabled, then failing cases. + BumpFee(transactionView, txid1, true /* expect disabled */, "not BIP 125 replaceable" /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, true /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, true /* expect disabled */, "already bumped" /* expected error */, false /* cancel */); + bitdb.Flush(true); bitdb.Reset(); } + +} + +void WalletTests::walletTests() +{ + TestSendCoins(); +} diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index d81188895b..233fc08772 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -293,13 +293,12 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco { COutPoint prevout = txin.prevout; - CCoins prev; - if(pcoinsTip->GetCoins(prevout.hash, prev)) + Coin prev; + if(pcoinsTip->GetCoin(prevout, prev)) { - if (prevout.n < prev.vout.size()) { strHTML += "<li>"; - const CTxOut &vout = prev.vout[prevout.n]; + const CTxOut &vout = prev.out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 4bb260aa58..0090b0c74b 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -246,13 +246,13 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) status.status = TransactionStatus::Confirmed; } } - + status.needsUpdate = false; } bool TransactionRecord::statusUpdateNeeded() { AssertLockHeld(cs_main); - return status.cur_num_blocks != chainActive.Height(); + return status.cur_num_blocks != chainActive.Height() || status.needsUpdate; } QString TransactionRecord::getTxID() const diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index 5aabbbffa8..59f681224f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -61,6 +61,8 @@ public: /** Current number of blocks (to know whether cached status is still valid) */ int cur_num_blocks; + + bool needsUpdate; }; /** UI model for a transaction. A core transaction can be represented by multiple UI transactions if it has diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 61466c8ed1..f27abc2104 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -168,6 +168,10 @@ public: case CT_UPDATED: // Miscellaneous updates -- nothing to do, status update will take care of this, and is only computed for // visible transactions. + for (int i = lowerIndex; i < upperIndex; i++) { + TransactionRecord *rec = &cachedWallet[i]; + rec->status.needsUpdate = true; + } break; } } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 9008c81634..e3e070b27f 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -136,10 +136,12 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa view->installEventFilter(this); transactionView = view; + transactionView->setObjectName("transactionView"); // Actions abandonAction = new QAction(tr("Abandon transaction"), this); bumpFeeAction = new QAction(tr("Increase transaction fee"), this); + bumpFeeAction->setObjectName("bumpFeeAction"); QAction *copyAddressAction = new QAction(tr("Copy address"), this); QAction *copyLabelAction = new QAction(tr("Copy label"), this); QAction *copyAmountAction = new QAction(tr("Copy amount"), this); @@ -150,6 +152,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa QAction *showDetailsAction = new QAction(tr("Show transaction details"), this); contextMenu = new QMenu(this); + contextMenu->setObjectName("contextMenu"); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyAmountAction); @@ -376,11 +379,11 @@ void TransactionView::contextualMenu(const QPoint &point) uint256 hash; hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->transactionCanBeAbandoned(hash)); - bumpFeeAction->setEnabled(model->transactionSignalsRBF(hash)); + bumpFeeAction->setEnabled(model->transactionCanBeBumped(hash)); if(index.isValid()) { - contextMenu->exec(QCursor::pos()); + contextMenu->popup(transactionView->viewport()->mapToGlobal(point)); } } @@ -416,7 +419,7 @@ void TransactionView::bumpFee() // Bump tx fee over the walletModel if (model->bumpFee(hash)) { // Update the table - model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); + model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, true); } } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 37af8abb38..8df0e481bd 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -68,14 +68,7 @@ CAmount WalletModel::getBalance(const CCoinControl *coinControl) const { if (coinControl) { - CAmount nBalance = 0; - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins, true, coinControl); - BOOST_FOREACH(const COutput& out, vCoins) - if(out.fSpendable) - nBalance += out.tx->tx->vout[out.i].nValue; - - return nBalance; + return wallet->GetAvailableBalance(coinControl); } return wallet->GetBalance(); @@ -600,38 +593,11 @@ bool WalletModel::isSpent(const COutPoint& outpoint) const // AvailableCoins + LockedCoins grouped by wallet address (put change in one group with wallet address) void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) const { - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins); - - LOCK2(cs_main, wallet->cs_wallet); // ListLockedCoins, mapWallet - std::vector<COutPoint> vLockedCoins; - wallet->ListLockedCoins(vLockedCoins); - - // add locked coins - BOOST_FOREACH(const COutPoint& outpoint, vLockedCoins) - { - if (!wallet->mapWallet.count(outpoint.hash)) continue; - int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); - if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true /* spendable */, true /* solvable */, true /* safe */); - if (outpoint.n < out.tx->tx->vout.size() && wallet->IsMine(out.tx->tx->vout[outpoint.n]) == ISMINE_SPENDABLE) - vCoins.push_back(out); - } - - BOOST_FOREACH(const COutput& out, vCoins) - { - COutput cout = out; - - while (wallet->IsChange(cout.tx->tx->vout[cout.i]) && cout.tx->tx->vin.size() > 0 && wallet->IsMine(cout.tx->tx->vin[0])) - { - if (!wallet->mapWallet.count(cout.tx->tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->tx->vin[0].prevout.hash], cout.tx->tx->vin[0].prevout.n, 0 /* depth */, true /* spendable */, true /* solvable */, true /* safe */); + for (auto& group : wallet->ListCoins()) { + auto& resultGroup = mapCoins[QString::fromStdString(CBitcoinAddress(group.first).ToString())]; + for (auto& coin : group.second) { + resultGroup.emplace_back(std::move(coin)); } - - CTxDestination address; - if(!out.fSpendable || !ExtractDestination(cout.tx->tx->vout[cout.i].scriptPubKey, address)) - continue; - mapCoins[QString::fromStdString(CBitcoinAddress(address).ToString())].push_back(out); } } @@ -661,11 +627,7 @@ void WalletModel::listLockedCoins(std::vector<COutPoint>& vOutpts) void WalletModel::loadReceiveRequests(std::vector<std::string>& vReceiveRequests) { - LOCK(wallet->cs_wallet); - BOOST_FOREACH(const PAIRTYPE(CTxDestination, CAddressBookData)& item, wallet->mapAddressBook) - BOOST_FOREACH(const PAIRTYPE(std::string, std::string)& item2, item.second.destdata) - if (item2.first.size() > 2 && item2.first.substr(0,2) == "rr") // receive request - vReceiveRequests.push_back(item2.second); + vReceiveRequests = wallet->GetDestValues("rr"); // receive request } bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t nId, const std::string &sRequest) @@ -685,11 +647,7 @@ bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t bool WalletModel::transactionCanBeAbandoned(uint256 hash) const { - LOCK2(cs_main, wallet->cs_wallet); - const CWalletTx *wtx = wallet->GetWalletTx(hash); - if (!wtx || wtx->isAbandoned() || wtx->GetDepthInMainChain() > 0 || wtx->InMempool()) - return false; - return true; + return wallet->TransactionCanBeAbandoned(hash); } bool WalletModel::abandonTransaction(uint256 hash) const @@ -698,13 +656,11 @@ bool WalletModel::abandonTransaction(uint256 hash) const return wallet->AbandonTransaction(hash); } -bool WalletModel::transactionSignalsRBF(uint256 hash) const +bool WalletModel::transactionCanBeBumped(uint256 hash) const { LOCK2(cs_main, wallet->cs_wallet); const CWalletTx *wtx = wallet->GetWalletTx(hash); - if (wtx && SignalsOptInRBF(*wtx)) - return true; - return false; + return wtx && SignalsOptInRBF(*wtx) && !wtx->mapValue.count("replaced_by_txid"); } bool WalletModel::bumpFee(uint256 hash) diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index df5acaf684..16b0caed4e 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -207,7 +207,7 @@ public: bool transactionCanBeAbandoned(uint256 hash) const; bool abandonTransaction(uint256 hash) const; - bool transactionSignalsRBF(uint256 hash) const; + bool transactionCanBeBumped(uint256 hash) const; bool bumpFee(uint256 hash); static bool isWalletEnabled(); diff --git a/src/random.cpp b/src/random.cpp index 3b9df3edaa..de7553c825 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -203,10 +203,43 @@ void GetRandBytes(unsigned char* buf, int num) } } +static void AddDataToRng(void* data, size_t len); + +void RandAddSeedSleep() +{ + int64_t nPerfCounter1 = GetPerformanceCounter(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + int64_t nPerfCounter2 = GetPerformanceCounter(); + + // Combine with and update state + AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1)); + AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2)); + + memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); + memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); +} + + static std::mutex cs_rng_state; static unsigned char rng_state[32] = {0}; static uint64_t rng_counter = 0; +static void AddDataToRng(void* data, size_t len) { + CSHA512 hasher; + hasher.Write((const unsigned char*)&len, sizeof(len)); + hasher.Write((const unsigned char*)data, len); + unsigned char buf[64]; + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + memory_cleanse(buf, 64); +} + void GetStrongRandBytes(unsigned char* out, int num) { assert(num <= 32); diff --git a/src/random.h b/src/random.h index 9551e1c461..6a63d57429 100644 --- a/src/random.h +++ b/src/random.h @@ -24,6 +24,13 @@ int GetRandInt(int nMax); uint256 GetRandHash(); /** + * Add a little bit of randomness to the output of GetStrongRangBytes. + * This sleeps for a millisecond, so should only be called when there is + * no other work to be done. + */ +void RandAddSeedSleep(); + +/** * Function to gather random data from multiple sources, failing whenever any * of those source fail to provide a result. */ diff --git a/src/rest.cpp b/src/rest.cpp index 7537ed4502..b08d7153b1 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -42,16 +42,19 @@ static const struct { }; struct CCoin { - uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE uint32_t nHeight; CTxOut out; ADD_SERIALIZE_METHODS; + CCoin() : nHeight(0) {} + CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} + template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTxVer); + uint32_t nTxVerDummy = 0; + READWRITE(nTxVerDummy); READWRITE(nHeight); READWRITE(out); } @@ -509,22 +512,11 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool for (size_t i = 0; i < vOutPoints.size(); i++) { - CCoins coins; - uint256 hash = vOutPoints[i].hash; bool hit = false; - if (view.GetCoins(hash, coins)) { - mempool.pruneSpent(hash, coins); - if (coins.IsAvailable(vOutPoints[i].n)) { - hit = true; - // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if - // n is valid but points to an already spent output (IsNull). - CCoin coin; - coin.nTxVer = coins.nVersion; - coin.nHeight = coins.nHeight; - coin.out = coins.vout.at(vOutPoints[i].n); - assert(!coin.out.IsNull()); - outs.push_back(coin); - } + Coin coin; + if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + hit = true; + outs.emplace_back(std::move(coin)); } hits.push_back(hit); @@ -568,7 +560,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); utxo.push_back(Pair("height", (int32_t)coin.nHeight)); utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 9d72a23e6d..b66c1c2b64 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -187,7 +187,7 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } - cond_blockchange.notify_all(); + cond_blockchange.notify_all(); } UniValue waitfornewblock(const JSONRPCRequest& request) @@ -781,13 +781,32 @@ struct CCoinsStats uint256 hashBlock; uint64_t nTransactions; uint64_t nTransactionOutputs; - uint64_t nSerializedSize; + uint64_t nBogoSize; uint256 hashSerialized; + uint64_t nDiskSize; CAmount nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} }; +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << VARINT(output.second.out.nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + + 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { @@ -800,32 +819,29 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - CAmount nTotalAmount = 0; + uint256 prevkey; + std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; i<coins.vout.size(); i++) { - const CTxOut &out = coins.vout[i]; - if (!out.IsNull()) { - stats.nTransactionOutputs++; - ss << VARINT(i+1); - ss << out; - nTotalAmount += out.nValue; - } + COutPoint key; + Coin coin; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - stats.nSerializedSize += 32 + pcursor->GetValueSize(); - ss << VARINT(0); + prevkey = key.hash; + outputs[key.n] = std::move(coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; + stats.nDiskSize = view->EstimateSize(); return true; } @@ -891,8 +907,9 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"bytes_serialized\": n, (numeric) The serialized size\n" - " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"bogosize\": n, (numeric) A meaningless metric for UTXO set size\n" + " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -909,8 +926,9 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); - ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); - ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); + ret.push_back(Pair("bogosize", (int64_t)stats.nBogoSize)); + ret.push_back(Pair("hash_serialized_2", stats.hashSerialized.GetHex())); + ret.push_back(Pair("disk_size", stats.nDiskSize)); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -963,37 +981,37 @@ UniValue gettxout(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int n = request.params[1].get_int(); + COutPoint out(hash, n); bool fMempool = true; if (request.params.size() > 2) fMempool = request.params[2].get_bool(); - CCoins coins; + Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins)) + if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { // TODO: filtering spent coins should be done by the CCoinsViewMemPool return NullUniValue; - mempool.pruneSpent(hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } } else { - if (!pcoinsTip->GetCoins(hash, coins)) + if (!pcoinsTip->GetCoin(out, coin)) { return NullUniValue; + } } - if (n<0 || (unsigned int)n>=coins.vout.size() || coins.vout[n].IsNull()) - return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); - if ((unsigned int)coins.nHeight == MEMPOOL_HEIGHT) + if (coin.nHeight == MEMPOOL_HEIGHT) { ret.push_back(Pair("confirmations", 0)); - else - ret.push_back(Pair("confirmations", pindex->nHeight - coins.nHeight + 1)); - ret.push_back(Pair("value", ValueFromAmount(coins.vout[n].nValue))); + } else { + ret.push_back(Pair("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1))); + } + ret.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); ret.push_back(Pair("scriptPubKey", o)); - ret.push_back(Pair("version", coins.nVersion)); - ret.push_back(Pair("coinbase", coins.fCoinBase)); + ret.push_back(Pair("coinbase", (bool)coin.fCoinBase)); return ret; } @@ -1074,6 +1092,17 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse rv.push_back(Pair("startTime", consensusParams.vDeployments[id].nStartTime)); rv.push_back(Pair("timeout", consensusParams.vDeployments[id].nTimeout)); rv.push_back(Pair("since", VersionBitsTipStateSinceHeight(consensusParams, id))); + if (THRESHOLD_STARTED == thresholdState) + { + UniValue statsUV(UniValue::VOBJ); + BIP9Stats statsStruct = VersionBitsTipStatistics(consensusParams, id); + statsUV.push_back(Pair("period", statsStruct.period)); + statsUV.push_back(Pair("threshold", statsStruct.threshold)); + statsUV.push_back(Pair("elapsed", statsStruct.elapsed)); + statsUV.push_back(Pair("count", statsStruct.count)); + statsUV.push_back(Pair("possible", statsStruct.possible)); + rv.push_back(Pair("statistics", statsUV)); + } return rv; } @@ -1119,7 +1148,14 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx (numeric) height of the first block to which the status applies\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " }\n" " }\n" " }\n" "}\n" @@ -1307,7 +1343,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) " \"bytes\": xxxxx, (numeric) Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted\n" " \"usage\": xxxxx, (numeric) Total memory usage for the mempool\n" " \"maxmempool\": xxxxx, (numeric) Maximum memory usage for the mempool\n" - " \"mempoolminfee\": xxxxx (numeric) Minimum fee for tx to be accepted\n" + " \"mempoolminfee\": xxxxx (numeric) Minimum feerate (" + CURRENCY_UNIT + " per KB) for tx to be accepted\n" "}\n" "\nExamples:\n" + HelpExampleCli("getmempoolinfo", "") diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 1c6f4082e1..cfe42ec7d8 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -308,7 +308,7 @@ static UniValue BIP22ValidationResult(const CValidationState& state) } std::string gbt_vb_name(const Consensus::DeploymentPos pos) { - const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; std::string s = vbinfo.name; if (!vbinfo.gbt_force) { s.insert(s.begin(), '!'); @@ -520,7 +520,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) // TODO: Maybe recheck connections/IBD and (if something wrong) send an expires-immediately template to stop miners? } - const struct BIP9DeploymentInfo& segwit_info = VersionBitsDeploymentInfo[Consensus::DEPLOYMENT_SEGWIT]; + const struct VBDeploymentInfo& segwit_info = VersionBitsDeploymentInfo[Consensus::DEPLOYMENT_SEGWIT]; // If the caller is indicating segwit support, then allow CreateNewBlock() // to select witness transactions, after segwit activates (otherwise // don't). @@ -634,7 +634,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) // FALL THROUGH to get vbavailable set... case THRESHOLD_STARTED: { - const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; vbavailable.push_back(Pair(gbt_vb_name(pos), consensusParams.vDeployments[pos].bit)); if (setClientRules.find(vbinfo.name) == setClientRules.end()) { if (!vbinfo.gbt_force) { @@ -647,7 +647,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) case THRESHOLD_ACTIVE: { // Add to rules only - const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; aRules.push_back(gbt_vb_name(pos)); if (setClientRules.find(vbinfo.name) == setClientRules.end()) { // Not supported by the client; make sure it's safe to proceed diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index cde5ae723b..10bf99eb38 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -76,7 +76,8 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " {\n" " \"id\": n, (numeric) Peer index\n" " \"addr\":\"host:port\", (string) The ip address and port of the peer\n" - " \"addrlocal\":\"ip:port\", (string) local address\n" + " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" + " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"relaytxes\":true|false, (boolean) Whether peer has asked us to relay transactions to it\n" " \"lastsend\": ttt, (numeric) The time in seconds since epoch (Jan 1 1970 GMT) of the last send\n" @@ -133,6 +134,8 @@ UniValue getpeerinfo(const JSONRPCRequest& request) obj.push_back(Pair("addr", stats.addrName)); if (!(stats.addrLocal.empty())) obj.push_back(Pair("addrlocal", stats.addrLocal)); + if (stats.addrBind.IsValid()) + obj.push_back(Pair("addrbind", stats.addrBind.ToString())); obj.push_back(Pair("services", strprintf("%016x", stats.nServices))); obj.push_back(Pair("relaytxes", stats.fRelayTxes)); obj.push_back(Pair("lastsend", stats.nLastSend)); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 683bb25246..e27c2a77c7 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -219,9 +219,10 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins(oneTxid, coins) && coins.nHeight > 0 && coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsSpent() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + pblockindex = chainActive[coin.nHeight]; + } } if (pblockindex == NULL) @@ -637,9 +638,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view BOOST_FOREACH(const CTxIn& txin, mergedTx.vin) { - const uint256& prevHash = txin.prevout.hash; - CCoins coins; - view.AccessCoins(prevHash); // this is certainly allowed to fail + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. } view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long @@ -691,24 +690,26 @@ UniValue signrawtransaction(const JSONRPCRequest& request) if (nOut < 0) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and not using the local wallet (private keys @@ -766,13 +767,13 @@ UniValue signrawtransaction(const JSONRPCRequest& request) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (coins == NULL || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -844,9 +845,12 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) nMaxRawTxFee = 0; CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsSpent(); + } bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 0c1cfa2718..923ba2c231 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,6 +4,7 @@ #include "scheduler.h" +#include "random.h" #include "reverselock.h" #include <assert.h> @@ -39,6 +40,11 @@ void CScheduler::serviceQueue() // is called. while (!shouldStop()) { try { + if (!shouldStop() && taskQueue.empty()) { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + // Use this chance to get a tiny bit more entropy + RandAddSeedSleep(); + } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. newTaskScheduled.wait(lock); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index f4e5313a78..222cff59ea 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1028,7 +1028,7 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& } // Size limits - if (stack.size() + altstack.size() > 1000) + if (stack.size() + altstack.size() > MAX_STACK_SIZE) return set_error(serror, SCRIPT_ERR_STACK_SIZE); } } @@ -1142,24 +1142,24 @@ public: uint256 GetPrevoutHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].prevout; + for (const auto& txin : txTo.vin) { + ss << txin.prevout; } return ss.GetHash(); } uint256 GetSequenceHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].nSequence; + for (const auto& txin : txTo.vin) { + ss << txin.nSequence; } return ss.GetHash(); } uint256 GetOutputsHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vout.size(); n++) { - ss << txTo.vout[n]; + for (const auto& txout : txTo.vout) { + ss << txout; } return ss.GetHash(); } diff --git a/src/script/script.h b/src/script/script.h index d7aaa04f80..95a5999a13 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -30,6 +30,9 @@ static const int MAX_PUBKEYS_PER_MULTISIG = 20; // Maximum script length in bytes static const int MAX_SCRIPT_SIZE = 10000; +// Maximum number of values on script interpreter stack +static const int MAX_STACK_SIZE = 1000; + // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. static const unsigned int LOCKTIME_THRESHOLD = 500000000; // Tue Nov 5 00:53:20 1985 UTC diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index e5657f7f31..c071fbe275 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -93,7 +93,10 @@ TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c -tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +if !ENABLE_COVERAGE +tests_CPPFLAGS += -DVERIFY +endif tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) tests_LDFLAGS = -static TESTS += tests @@ -102,7 +105,10 @@ endif if USE_EXHAUSTIVE_TESTS noinst_PROGRAMS += exhaustive_tests exhaustive_tests_SOURCES = src/tests_exhaustive.c -exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src $(SECP_INCLUDES) +exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src $(SECP_INCLUDES) +if !ENABLE_COVERAGE +exhaustive_tests_CPPFLAGS += -DVERIFY +endif exhaustive_tests_LDADD = $(SECP_LIBS) exhaustive_tests_LDFLAGS = -static TESTS += exhaustive_tests diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index ec50ffe3a2..e5fcbcb4ed 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -20,7 +20,7 @@ AC_PATH_TOOL(STRIP, strip) AX_PROG_CC_FOR_BUILD if test "x$CFLAGS" = "x"; then - CFLAGS="-O3 -g" + CFLAGS="-g" fi AM_PROG_CC_C_O @@ -89,6 +89,11 @@ AC_ARG_ENABLE(benchmark, [use_benchmark=$enableval], [use_benchmark=no]) +AC_ARG_ENABLE(coverage, + AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis]), + [enable_coverage=$enableval], + [enable_coverage=no]) + AC_ARG_ENABLE(tests, AS_HELP_STRING([--enable-tests],[compile tests (default is yes)]), [use_tests=$enableval], @@ -154,6 +159,14 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[void myfunc() {__builtin_expect(0,0);}]])], [ AC_MSG_RESULT([no]) ]) +if test x"$enable_coverage" = x"yes"; then + AC_DEFINE(COVERAGE, 1, [Define this symbol to compile out all VERIFY code]) + CFLAGS="$CFLAGS -O0 --coverage" + LDFLAGS="--coverage" +else + CFLAGS="$CFLAGS -O3" +fi + if test x"$use_ecmult_static_precomputation" != x"no"; then save_cross_compiling=$cross_compiling cross_compiling=no @@ -434,6 +447,7 @@ AC_MSG_NOTICE([Using field implementation: $set_field]) AC_MSG_NOTICE([Using bignum implementation: $set_bignum]) AC_MSG_NOTICE([Using scalar implementation: $set_scalar]) AC_MSG_NOTICE([Using endomorphism optimizations: $use_endomorphism]) +AC_MSG_NOTICE([Building for coverage analysis: $enable_coverage]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) AC_MSG_NOTICE([Building ECDSA pubkey recovery module: $enable_module_recovery]) AC_MSG_NOTICE([Using jni: $use_jni]) @@ -460,6 +474,7 @@ AC_SUBST(SECP_INCLUDES) AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) +AM_CONDITIONAL([ENABLE_COVERAGE], [test x"$enable_coverage" = x"yes"]) AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index f268e309d0..fc4c5cefbb 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -163,6 +163,8 @@ typedef int (*secp256k1_nonce_function)( * * Returns: a newly created context object. * In: flags: which parts of the context to initialize. + * + * See also secp256k1_context_randomize. */ SECP256K1_API secp256k1_context* secp256k1_context_create( unsigned int flags @@ -485,6 +487,28 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( const unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Negates a private key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( + const secp256k1_context* ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Negates a public key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_negate( + const secp256k1_context* ctx, + secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + /** Tweak a private key by adding tweak to it. * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for * uniformly random 32-byte arrays, or if the resulting private key @@ -543,11 +567,24 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( const unsigned char *tweak ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Updates the context randomization. +/** Updates the context randomization to protect against side-channel leakage. * Returns: 1: randomization successfully updated * 0: error * Args: ctx: pointer to a context object (cannot be NULL) * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state) + * + * While secp256k1 code is written to be constant-time no matter what secret + * values are, it's possible that a future compiler may output code which isn't, + * and also that the CPU may not emit the same radio frequencies or draw the same + * amount power for all values. + * + * This function provides a seed which is combined into the blinding value: that + * blinding value is added before each multiplication (and removed afterwards) so + * that it does not affect function results, but shields against attacks which + * rely on any input-dependent behaviour. + * + * You should call this after secp256k1_context_create or + * secp256k1_context_clone, and may call this repeatedly afterwards. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( secp256k1_context* ctx, diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index 3a71b4aafa..d67f08a426 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -23,7 +23,7 @@ void print_number(double x) { if (y < 0.0) { y = -y; } - while (y < 100.0) { + while (y > 0 && y < 100.0) { y *= 10.0; c++; } diff --git a/src/secp256k1/src/bench_schnorr_verify.c b/src/secp256k1/src/bench_schnorr_verify.c deleted file mode 100644 index 5f137dda23..0000000000 --- a/src/secp256k1/src/bench_schnorr_verify.c +++ /dev/null @@ -1,73 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#include <stdio.h> -#include <string.h> - -#include "include/secp256k1.h" -#include "include/secp256k1_schnorr.h" -#include "util.h" -#include "bench.h" - -typedef struct { - unsigned char key[32]; - unsigned char sig[64]; - unsigned char pubkey[33]; - size_t pubkeylen; -} benchmark_schnorr_sig_t; - -typedef struct { - secp256k1_context *ctx; - unsigned char msg[32]; - benchmark_schnorr_sig_t sigs[64]; - int numsigs; -} benchmark_schnorr_verify_t; - -static void benchmark_schnorr_init(void* arg) { - int i, k; - benchmark_schnorr_verify_t* data = (benchmark_schnorr_verify_t*)arg; - - for (i = 0; i < 32; i++) { - data->msg[i] = 1 + i; - } - for (k = 0; k < data->numsigs; k++) { - secp256k1_pubkey pubkey; - for (i = 0; i < 32; i++) { - data->sigs[k].key[i] = 33 + i + k; - } - secp256k1_schnorr_sign(data->ctx, data->sigs[k].sig, data->msg, data->sigs[k].key, NULL, NULL); - data->sigs[k].pubkeylen = 33; - CHECK(secp256k1_ec_pubkey_create(data->ctx, &pubkey, data->sigs[k].key)); - CHECK(secp256k1_ec_pubkey_serialize(data->ctx, data->sigs[k].pubkey, &data->sigs[k].pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED)); - } -} - -static void benchmark_schnorr_verify(void* arg) { - int i; - benchmark_schnorr_verify_t* data = (benchmark_schnorr_verify_t*)arg; - - for (i = 0; i < 20000 / data->numsigs; i++) { - secp256k1_pubkey pubkey; - data->sigs[0].sig[(i >> 8) % 64] ^= (i & 0xFF); - CHECK(secp256k1_ec_pubkey_parse(data->ctx, &pubkey, data->sigs[0].pubkey, data->sigs[0].pubkeylen)); - CHECK(secp256k1_schnorr_verify(data->ctx, data->sigs[0].sig, data->msg, &pubkey) == ((i & 0xFF) == 0)); - data->sigs[0].sig[(i >> 8) % 64] ^= (i & 0xFF); - } -} - - - -int main(void) { - benchmark_schnorr_verify_t data; - - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - - data.numsigs = 1; - run_benchmark("schnorr_verify", benchmark_schnorr_verify, benchmark_schnorr_init, NULL, &data, 10, 20000); - - secp256k1_context_destroy(data.ctx); - return 0; -} diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index 9a42e519bd..453bb11880 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -225,14 +225,12 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const #if defined(EXHAUSTIVE_TEST_ORDER) { secp256k1_scalar computed_r; - int overflow = 0; secp256k1_ge pr_ge; secp256k1_ge_set_gej(&pr_ge, &pr); secp256k1_fe_normalize(&pr_ge.x); secp256k1_fe_get_b32(c, &pr_ge.x); - secp256k1_scalar_set_b32(&computed_r, c, &overflow); - /* we fully expect overflow */ + secp256k1_scalar_set_b32(&computed_r, c, NULL); return secp256k1_scalar_eq(sigr, &computed_r); } #else @@ -285,14 +283,10 @@ static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, sec secp256k1_fe_normalize(&r.y); secp256k1_fe_get_b32(b, &r.x); secp256k1_scalar_set_b32(sigr, b, &overflow); - if (secp256k1_scalar_is_zero(sigr)) { - /* P.x = order is on the curve, so technically sig->r could end up zero, which would be an invalid signature. - * This branch is cryptographically unreachable as hitting it requires finding the discrete log of P.x = N. - */ - secp256k1_gej_clear(&rp); - secp256k1_ge_clear(&r); - return 0; - } + /* These two conditions should be checked before calling */ + VERIFY_CHECK(!secp256k1_scalar_is_zero(sigr)); + VERIFY_CHECK(overflow == 0); + if (recid) { /* The overflow condition is cryptographically unreachable as hitting it requires finding the discrete log * of some P where P.x >= order, and only 1 in about 2^127 points meet this criteria. diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 7b8c079608..234c13a644 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -38,10 +38,6 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } VERIFY_CHECK(r == 1); } -#else -static void secp256k1_fe_verify(const secp256k1_fe *a) { - (void)a; -} #endif static void secp256k1_fe_normalize(secp256k1_fe *r) { @@ -325,17 +321,17 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int i; - r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; - r->n[5] = r->n[6] = r->n[7] = r->n[8] = r->n[9] = 0; - for (i=0; i<32; i++) { - int j; - for (j=0; j<4; j++) { - int limb = (8*i+2*j)/26; - int shift = (8*i+2*j)%26; - r->n[limb] |= (uint32_t)((a[31-i] >> (2*j)) & 0x3) << shift; - } - } + r->n[0] = (uint32_t)a[31] | ((uint32_t)a[30] << 8) | ((uint32_t)a[29] << 16) | ((uint32_t)(a[28] & 0x3) << 24); + r->n[1] = (uint32_t)((a[28] >> 2) & 0x3f) | ((uint32_t)a[27] << 6) | ((uint32_t)a[26] << 14) | ((uint32_t)(a[25] & 0xf) << 22); + r->n[2] = (uint32_t)((a[25] >> 4) & 0xf) | ((uint32_t)a[24] << 4) | ((uint32_t)a[23] << 12) | ((uint32_t)(a[22] & 0x3f) << 20); + r->n[3] = (uint32_t)((a[22] >> 6) & 0x3) | ((uint32_t)a[21] << 2) | ((uint32_t)a[20] << 10) | ((uint32_t)a[19] << 18); + r->n[4] = (uint32_t)a[18] | ((uint32_t)a[17] << 8) | ((uint32_t)a[16] << 16) | ((uint32_t)(a[15] & 0x3) << 24); + r->n[5] = (uint32_t)((a[15] >> 2) & 0x3f) | ((uint32_t)a[14] << 6) | ((uint32_t)a[13] << 14) | ((uint32_t)(a[12] & 0xf) << 22); + r->n[6] = (uint32_t)((a[12] >> 4) & 0xf) | ((uint32_t)a[11] << 4) | ((uint32_t)a[10] << 12) | ((uint32_t)(a[9] & 0x3f) << 20); + r->n[7] = (uint32_t)((a[9] >> 6) & 0x3) | ((uint32_t)a[8] << 2) | ((uint32_t)a[7] << 10) | ((uint32_t)a[6] << 18); + r->n[8] = (uint32_t)a[5] | ((uint32_t)a[4] << 8) | ((uint32_t)a[3] << 16) | ((uint32_t)(a[2] & 0x3) << 24); + r->n[9] = (uint32_t)((a[2] >> 2) & 0x3f) | ((uint32_t)a[1] << 6) | ((uint32_t)a[0] << 14); + if (r->n[9] == 0x3FFFFFUL && (r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL && (r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL) { return 0; } @@ -349,21 +345,42 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - int i; #ifdef VERIFY VERIFY_CHECK(a->normalized); secp256k1_fe_verify(a); #endif - for (i=0; i<32; i++) { - int j; - int c = 0; - for (j=0; j<4; j++) { - int limb = (8*i+2*j)/26; - int shift = (8*i+2*j)%26; - c |= ((a->n[limb] >> shift) & 0x3) << (2 * j); - } - r[31-i] = c; - } + r[0] = (a->n[9] >> 14) & 0xff; + r[1] = (a->n[9] >> 6) & 0xff; + r[2] = ((a->n[9] & 0x3F) << 2) | ((a->n[8] >> 24) & 0x3); + r[3] = (a->n[8] >> 16) & 0xff; + r[4] = (a->n[8] >> 8) & 0xff; + r[5] = a->n[8] & 0xff; + r[6] = (a->n[7] >> 18) & 0xff; + r[7] = (a->n[7] >> 10) & 0xff; + r[8] = (a->n[7] >> 2) & 0xff; + r[9] = ((a->n[7] & 0x3) << 6) | ((a->n[6] >> 20) & 0x3f); + r[10] = (a->n[6] >> 12) & 0xff; + r[11] = (a->n[6] >> 4) & 0xff; + r[12] = ((a->n[6] & 0xf) << 4) | ((a->n[5] >> 22) & 0xf); + r[13] = (a->n[5] >> 14) & 0xff; + r[14] = (a->n[5] >> 6) & 0xff; + r[15] = ((a->n[5] & 0x3f) << 2) | ((a->n[4] >> 24) & 0x3); + r[16] = (a->n[4] >> 16) & 0xff; + r[17] = (a->n[4] >> 8) & 0xff; + r[18] = a->n[4] & 0xff; + r[19] = (a->n[3] >> 18) & 0xff; + r[20] = (a->n[3] >> 10) & 0xff; + r[21] = (a->n[3] >> 2) & 0xff; + r[22] = ((a->n[3] & 0x3) << 6) | ((a->n[2] >> 20) & 0x3f); + r[23] = (a->n[2] >> 12) & 0xff; + r[24] = (a->n[2] >> 4) & 0xff; + r[25] = ((a->n[2] & 0xf) << 4) | ((a->n[1] >> 22) & 0xf); + r[26] = (a->n[1] >> 14) & 0xff; + r[27] = (a->n[1] >> 6) & 0xff; + r[28] = ((a->n[1] & 0x3f) << 2) | ((a->n[0] >> 24) & 0x3); + r[29] = (a->n[0] >> 16) & 0xff; + r[30] = (a->n[0] >> 8) & 0xff; + r[31] = a->n[0] & 0xff; } SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index 7a99eb21ec..8e8b286baf 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -49,10 +49,6 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } VERIFY_CHECK(r == 1); } -#else -static void secp256k1_fe_verify(const secp256k1_fe *a) { - (void)a; -} #endif static void secp256k1_fe_normalize(secp256k1_fe *r) { @@ -288,16 +284,40 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int i; - r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; - for (i=0; i<32; i++) { - int j; - for (j=0; j<2; j++) { - int limb = (8*i+4*j)/52; - int shift = (8*i+4*j)%52; - r->n[limb] |= (uint64_t)((a[31-i] >> (4*j)) & 0xF) << shift; - } - } + r->n[0] = (uint64_t)a[31] + | ((uint64_t)a[30] << 8) + | ((uint64_t)a[29] << 16) + | ((uint64_t)a[28] << 24) + | ((uint64_t)a[27] << 32) + | ((uint64_t)a[26] << 40) + | ((uint64_t)(a[25] & 0xF) << 48); + r->n[1] = (uint64_t)((a[25] >> 4) & 0xF) + | ((uint64_t)a[24] << 4) + | ((uint64_t)a[23] << 12) + | ((uint64_t)a[22] << 20) + | ((uint64_t)a[21] << 28) + | ((uint64_t)a[20] << 36) + | ((uint64_t)a[19] << 44); + r->n[2] = (uint64_t)a[18] + | ((uint64_t)a[17] << 8) + | ((uint64_t)a[16] << 16) + | ((uint64_t)a[15] << 24) + | ((uint64_t)a[14] << 32) + | ((uint64_t)a[13] << 40) + | ((uint64_t)(a[12] & 0xF) << 48); + r->n[3] = (uint64_t)((a[12] >> 4) & 0xF) + | ((uint64_t)a[11] << 4) + | ((uint64_t)a[10] << 12) + | ((uint64_t)a[9] << 20) + | ((uint64_t)a[8] << 28) + | ((uint64_t)a[7] << 36) + | ((uint64_t)a[6] << 44); + r->n[4] = (uint64_t)a[5] + | ((uint64_t)a[4] << 8) + | ((uint64_t)a[3] << 16) + | ((uint64_t)a[2] << 24) + | ((uint64_t)a[1] << 32) + | ((uint64_t)a[0] << 40); if (r->n[4] == 0x0FFFFFFFFFFFFULL && (r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL && r->n[0] >= 0xFFFFEFFFFFC2FULL) { return 0; } @@ -311,21 +331,42 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - int i; #ifdef VERIFY VERIFY_CHECK(a->normalized); secp256k1_fe_verify(a); #endif - for (i=0; i<32; i++) { - int j; - int c = 0; - for (j=0; j<2; j++) { - int limb = (8*i+4*j)/52; - int shift = (8*i+4*j)%52; - c |= ((a->n[limb] >> shift) & 0xF) << (4 * j); - } - r[31-i] = c; - } + r[0] = (a->n[4] >> 40) & 0xFF; + r[1] = (a->n[4] >> 32) & 0xFF; + r[2] = (a->n[4] >> 24) & 0xFF; + r[3] = (a->n[4] >> 16) & 0xFF; + r[4] = (a->n[4] >> 8) & 0xFF; + r[5] = a->n[4] & 0xFF; + r[6] = (a->n[3] >> 44) & 0xFF; + r[7] = (a->n[3] >> 36) & 0xFF; + r[8] = (a->n[3] >> 28) & 0xFF; + r[9] = (a->n[3] >> 20) & 0xFF; + r[10] = (a->n[3] >> 12) & 0xFF; + r[11] = (a->n[3] >> 4) & 0xFF; + r[12] = ((a->n[2] >> 48) & 0xF) | ((a->n[3] & 0xF) << 4); + r[13] = (a->n[2] >> 40) & 0xFF; + r[14] = (a->n[2] >> 32) & 0xFF; + r[15] = (a->n[2] >> 24) & 0xFF; + r[16] = (a->n[2] >> 16) & 0xFF; + r[17] = (a->n[2] >> 8) & 0xFF; + r[18] = a->n[2] & 0xFF; + r[19] = (a->n[1] >> 44) & 0xFF; + r[20] = (a->n[1] >> 36) & 0xFF; + r[21] = (a->n[1] >> 28) & 0xFF; + r[22] = (a->n[1] >> 20) & 0xFF; + r[23] = (a->n[1] >> 12) & 0xFF; + r[24] = (a->n[1] >> 4) & 0xFF; + r[25] = ((a->n[0] >> 48) & 0xF) | ((a->n[1] & 0xF) << 4); + r[26] = (a->n[0] >> 40) & 0xFF; + r[27] = (a->n[0] >> 32) & 0xFF; + r[28] = (a->n[0] >> 24) & 0xFF; + r[29] = (a->n[0] >> 16) & 0xFF; + r[30] = (a->n[0] >> 8) & 0xFF; + r[31] = a->n[0] & 0xFF; } SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 2e192b62fd..7d723532ff 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -200,12 +200,6 @@ static void secp256k1_gej_set_infinity(secp256k1_gej *r) { secp256k1_fe_clear(&r->z); } -static void secp256k1_ge_set_infinity(secp256k1_ge *r) { - r->infinity = 1; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); -} - static void secp256k1_gej_clear(secp256k1_gej *r) { r->infinity = 0; secp256k1_fe_clear(&r->x); diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index c23e4f82f7..9e30fb73dd 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -16,10 +16,10 @@ int secp256k1_ecdh(const secp256k1_context* ctx, unsigned char *result, const se secp256k1_gej res; secp256k1_ge pt; secp256k1_scalar s; + VERIFY_CHECK(ctx != NULL); ARG_CHECK(result != NULL); ARG_CHECK(point != NULL); ARG_CHECK(scalar != NULL); - (void)ctx; secp256k1_pubkey_load(ctx, &pt, point); secp256k1_scalar_set_b32(&s, scalar, &overflow); diff --git a/src/secp256k1/src/modules/ecdh/tests_impl.h b/src/secp256k1/src/modules/ecdh/tests_impl.h index 7badc9033f..85a5d0a9a6 100644 --- a/src/secp256k1/src/modules/ecdh/tests_impl.h +++ b/src/secp256k1/src/modules/ecdh/tests_impl.h @@ -7,6 +7,35 @@ #ifndef _SECP256K1_MODULE_ECDH_TESTS_ #define _SECP256K1_MODULE_ECDH_TESTS_ +void test_ecdh_api(void) { + /* Setup context that just counts errors */ + secp256k1_context *tctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_pubkey point; + unsigned char res[32]; + unsigned char s_one[32] = { 0 }; + int32_t ecount = 0; + s_one[31] = 1; + + secp256k1_context_set_error_callback(tctx, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(tctx, counting_illegal_callback_fn, &ecount); + CHECK(secp256k1_ec_pubkey_create(tctx, &point, s_one) == 1); + + /* Check all NULLs are detected */ + CHECK(secp256k1_ecdh(tctx, res, &point, s_one) == 1); + CHECK(ecount == 0); + CHECK(secp256k1_ecdh(tctx, NULL, &point, s_one) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdh(tctx, res, NULL, s_one) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdh(tctx, res, &point, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdh(tctx, res, &point, s_one) == 1); + CHECK(ecount == 3); + + /* Cleanup */ + secp256k1_context_destroy(tctx); +} + void test_ecdh_generator_basepoint(void) { unsigned char s_one[32] = { 0 }; secp256k1_pubkey point[2]; @@ -68,6 +97,7 @@ void test_bad_scalar(void) { } void run_ecdh_tests(void) { + test_ecdh_api(); test_ecdh_generator_basepoint(); test_bad_scalar(); } diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index 86f2f0cb2b..c6fbe23981 100755 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -179,7 +179,7 @@ int secp256k1_ecdsa_recover(const secp256k1_context* ctx, secp256k1_pubkey *pubk ARG_CHECK(pubkey != NULL); secp256k1_ecdsa_recoverable_signature_load(ctx, &r, &s, &recid, signature); - ARG_CHECK(recid >= 0 && recid < 4); + VERIFY_CHECK(recid >= 0 && recid < 4); /* should have been caught in parse_compact */ secp256k1_scalar_set_b32(&m, msg32, NULL); if (secp256k1_ecdsa_sig_recover(&ctx->ecmult_ctx, &r, &s, &q, &m, recid)) { secp256k1_pubkey_save(pubkey, &q); diff --git a/src/secp256k1/src/modules/recovery/tests_impl.h b/src/secp256k1/src/modules/recovery/tests_impl.h index 8932d5f0af..765c7dd81e 100644 --- a/src/secp256k1/src/modules/recovery/tests_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_impl.h @@ -7,6 +7,146 @@ #ifndef _SECP256K1_MODULE_RECOVERY_TESTS_ #define _SECP256K1_MODULE_RECOVERY_TESTS_ +static int recovery_test_nonce_function(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *algo16, void *data, unsigned int counter) { + (void) msg32; + (void) key32; + (void) algo16; + (void) data; + + /* On the first run, return 0 to force a second run */ + if (counter == 0) { + memset(nonce32, 0, 32); + return 1; + } + /* On the second run, return an overflow to force a third run */ + if (counter == 1) { + memset(nonce32, 0xff, 32); + return 1; + } + /* On the next run, return a valid nonce, but flip a coin as to whether or not to fail signing. */ + memset(nonce32, 1, 32); + return secp256k1_rand_bits(1); +} + +void test_ecdsa_recovery_api(void) { + /* Setup contexts that just count errors */ + secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + secp256k1_pubkey pubkey; + secp256k1_pubkey recpubkey; + secp256k1_ecdsa_signature normal_sig; + secp256k1_ecdsa_recoverable_signature recsig; + unsigned char privkey[32] = { 1 }; + unsigned char message[32] = { 2 }; + int32_t ecount = 0; + int recid = 0; + unsigned char sig[74]; + unsigned char zero_privkey[32] = { 0 }; + unsigned char over_privkey[32] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + + secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(both, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(both, counting_illegal_callback_fn, &ecount); + + /* Construct and verify corresponding public key. */ + CHECK(secp256k1_ec_seckey_verify(ctx, privkey) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey, privkey) == 1); + + /* Check bad contexts and NULLs for signing */ + ecount = 0; + CHECK(secp256k1_ecdsa_sign_recoverable(none, &recsig, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(sign, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(vrfy, &recsig, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_sign_recoverable(both, NULL, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, NULL, privkey, NULL, NULL) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, NULL, NULL, NULL) == 0); + CHECK(ecount == 5); + /* This will fail or succeed randomly, and in either case will not ARG_CHECK failure */ + secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, recovery_test_nonce_function, NULL); + CHECK(ecount == 5); + /* These will all fail, but not in ARG_CHECK way */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, zero_privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, over_privkey, NULL, NULL) == 0); + /* This one will succeed. */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 5); + + /* Check signing with a goofy nonce function */ + + /* Check bad contexts and NULLs for recovery */ + ecount = 0; + CHECK(secp256k1_ecdsa_recover(none, &recpubkey, &recsig, message) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recover(sign, &recpubkey, &recsig, message) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(vrfy, &recpubkey, &recsig, message) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, message) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(both, NULL, &recsig, message) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, NULL, message) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, NULL) == 0); + CHECK(ecount == 5); + + /* Check NULLs for conversion */ + CHECK(secp256k1_ecdsa_sign(both, &normal_sig, message, privkey, NULL, NULL) == 1); + ecount = 0; + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, NULL, &recsig) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, &recsig) == 1); + + /* Check NULLs for de/serialization */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + ecount = 0; + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, NULL, &recid, &recsig) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, NULL, &recsig) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, &recsig) == 1); + + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, NULL, sig, recid) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, NULL, recid) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, -1) == 0); + CHECK(ecount == 6); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, 5) == 0); + CHECK(ecount == 7); + /* overflow in signature will fail but not affect ecount */ + memcpy(sig, over_privkey, 32); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, recid) == 0); + CHECK(ecount == 7); + + /* cleanup */ + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(vrfy); + secp256k1_context_destroy(both); +} + void test_ecdsa_recovery_end_to_end(void) { unsigned char extra[32] = {0x00}; unsigned char privkey[32]; @@ -241,6 +381,9 @@ void test_ecdsa_recovery_edge_cases(void) { void run_recovery_tests(void) { int i; + for (i = 0; i < count; i++) { + test_ecdsa_recovery_api(); + } for (i = 0; i < 64*count; i++) { test_ecdsa_recovery_end_to_end(); } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index f5b2376407..2690d86558 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -66,88 +66,79 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar #else secp256k1_scalar *t; int i; - /* First compute x ^ (2^N - 1) for some values of N. */ - secp256k1_scalar x2, x3, x4, x6, x7, x8, x15, x30, x60, x120, x127; + /* First compute xN as x ^ (2^N - 1) for some values of N, + * and uM as x ^ M for some values of M. */ + secp256k1_scalar x2, x3, x6, x8, x14, x28, x56, x112, x126; + secp256k1_scalar u2, u5, u9, u11, u13; - secp256k1_scalar_sqr(&x2, x); - secp256k1_scalar_mul(&x2, &x2, x); + secp256k1_scalar_sqr(&u2, x); + secp256k1_scalar_mul(&x2, &u2, x); + secp256k1_scalar_mul(&u5, &u2, &x2); + secp256k1_scalar_mul(&x3, &u5, &u2); + secp256k1_scalar_mul(&u9, &x3, &u2); + secp256k1_scalar_mul(&u11, &u9, &u2); + secp256k1_scalar_mul(&u13, &u11, &u2); - secp256k1_scalar_sqr(&x3, &x2); - secp256k1_scalar_mul(&x3, &x3, x); - - secp256k1_scalar_sqr(&x4, &x3); - secp256k1_scalar_mul(&x4, &x4, x); - - secp256k1_scalar_sqr(&x6, &x4); + secp256k1_scalar_sqr(&x6, &u13); secp256k1_scalar_sqr(&x6, &x6); - secp256k1_scalar_mul(&x6, &x6, &x2); - - secp256k1_scalar_sqr(&x7, &x6); - secp256k1_scalar_mul(&x7, &x7, x); + secp256k1_scalar_mul(&x6, &x6, &u11); - secp256k1_scalar_sqr(&x8, &x7); - secp256k1_scalar_mul(&x8, &x8, x); + secp256k1_scalar_sqr(&x8, &x6); + secp256k1_scalar_sqr(&x8, &x8); + secp256k1_scalar_mul(&x8, &x8, &x2); - secp256k1_scalar_sqr(&x15, &x8); - for (i = 0; i < 6; i++) { - secp256k1_scalar_sqr(&x15, &x15); + secp256k1_scalar_sqr(&x14, &x8); + for (i = 0; i < 5; i++) { + secp256k1_scalar_sqr(&x14, &x14); } - secp256k1_scalar_mul(&x15, &x15, &x7); + secp256k1_scalar_mul(&x14, &x14, &x6); - secp256k1_scalar_sqr(&x30, &x15); - for (i = 0; i < 14; i++) { - secp256k1_scalar_sqr(&x30, &x30); + secp256k1_scalar_sqr(&x28, &x14); + for (i = 0; i < 13; i++) { + secp256k1_scalar_sqr(&x28, &x28); } - secp256k1_scalar_mul(&x30, &x30, &x15); + secp256k1_scalar_mul(&x28, &x28, &x14); - secp256k1_scalar_sqr(&x60, &x30); - for (i = 0; i < 29; i++) { - secp256k1_scalar_sqr(&x60, &x60); + secp256k1_scalar_sqr(&x56, &x28); + for (i = 0; i < 27; i++) { + secp256k1_scalar_sqr(&x56, &x56); } - secp256k1_scalar_mul(&x60, &x60, &x30); + secp256k1_scalar_mul(&x56, &x56, &x28); - secp256k1_scalar_sqr(&x120, &x60); - for (i = 0; i < 59; i++) { - secp256k1_scalar_sqr(&x120, &x120); + secp256k1_scalar_sqr(&x112, &x56); + for (i = 0; i < 55; i++) { + secp256k1_scalar_sqr(&x112, &x112); } - secp256k1_scalar_mul(&x120, &x120, &x60); + secp256k1_scalar_mul(&x112, &x112, &x56); - secp256k1_scalar_sqr(&x127, &x120); - for (i = 0; i < 6; i++) { - secp256k1_scalar_sqr(&x127, &x127); + secp256k1_scalar_sqr(&x126, &x112); + for (i = 0; i < 13; i++) { + secp256k1_scalar_sqr(&x126, &x126); } - secp256k1_scalar_mul(&x127, &x127, &x7); + secp256k1_scalar_mul(&x126, &x126, &x14); - /* Then accumulate the final result (t starts at x127). */ - t = &x127; - for (i = 0; i < 2; i++) { /* 0 */ + /* Then accumulate the final result (t starts at x126). */ + t = &x126; + for (i = 0; i < 3; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ + for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 3; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } @@ -156,38 +147,26 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 4; i++) { /* 00 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ + for (i = 0; i < 3; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &x3); /* 111 */ for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) { /* 000 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ + for (i = 0; i < 6; i++) { /* 000 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ for (i = 0; i < 10; i++) { /* 0000000 */ secp256k1_scalar_sqr(t, t); } @@ -200,50 +179,34 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x8); /* 11111111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 5; i++) { /* 000 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 4; i++) { /* 00 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 5; i++) { secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) { /* 0 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 8; i++) { /* 000000 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 10; i++) { /* 000000 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) { /* 00 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ for (i = 0; i < 6; i++) { /* 00000 */ secp256k1_scalar_sqr(t, t); } diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index fb8b882faa..4f8c01655b 100755..100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -424,6 +424,33 @@ int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *p return ret; } +int secp256k1_ec_privkey_negate(const secp256k1_context* ctx, unsigned char *seckey) { + secp256k1_scalar sec; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(seckey != NULL); + + secp256k1_scalar_set_b32(&sec, seckey, NULL); + secp256k1_scalar_negate(&sec, &sec); + secp256k1_scalar_get_b32(seckey, &sec); + + return 1; +} + +int secp256k1_ec_pubkey_negate(const secp256k1_context* ctx, secp256k1_pubkey *pubkey) { + int ret = 0; + secp256k1_ge p; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + + ret = secp256k1_pubkey_load(ctx, &p, pubkey); + memset(pubkey, 0, sizeof(*pubkey)); + if (ret) { + secp256k1_ge_neg(&p, &p); + secp256k1_pubkey_save(pubkey, &p); + } + return ret; +} + int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar term; secp256k1_scalar sec; @@ -552,10 +579,6 @@ int secp256k1_ec_pubkey_combine(const secp256k1_context* ctx, secp256k1_pubkey * # include "modules/ecdh/main_impl.h" #endif -#ifdef ENABLE_MODULE_SCHNORR -# include "modules/schnorr/main_impl.h" -#endif - #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/main_impl.h" #endif diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 9ae7d30281..3d9bd5ebb4 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -10,6 +10,7 @@ #include <stdio.h> #include <stdlib.h> +#include <string.h> #include <time.h> @@ -135,6 +136,7 @@ void random_scalar_order(secp256k1_scalar *num) { void run_context_tests(void) { secp256k1_pubkey pubkey; + secp256k1_pubkey zero_pubkey; secp256k1_ecdsa_signature sig; unsigned char ctmp[32]; int32_t ecount; @@ -149,6 +151,8 @@ void run_context_tests(void) { secp256k1_scalar msg, key, nonce; secp256k1_scalar sigr, sigs; + memset(&zero_pubkey, 0, sizeof(zero_pubkey)); + ecount = 0; ecount2 = 10; secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); @@ -201,12 +205,20 @@ void run_context_tests(void) { CHECK(ecount == 2); CHECK(secp256k1_ec_pubkey_tweak_mul(sign, &pubkey, ctmp) == 0); CHECK(ecount2 == 13); - CHECK(secp256k1_ec_pubkey_tweak_mul(vrfy, &pubkey, ctmp) == 1); + CHECK(secp256k1_ec_pubkey_negate(vrfy, &pubkey) == 1); CHECK(ecount == 2); - CHECK(secp256k1_context_randomize(vrfy, ctmp) == 0); + CHECK(secp256k1_ec_pubkey_negate(sign, &pubkey) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ec_pubkey_negate(sign, NULL) == 0); + CHECK(ecount2 == 14); + CHECK(secp256k1_ec_pubkey_negate(vrfy, &zero_pubkey) == 0); CHECK(ecount == 3); + CHECK(secp256k1_ec_pubkey_tweak_mul(vrfy, &pubkey, ctmp) == 1); + CHECK(ecount == 3); + CHECK(secp256k1_context_randomize(vrfy, ctmp) == 0); + CHECK(ecount == 4); CHECK(secp256k1_context_randomize(sign, NULL) == 1); - CHECK(ecount2 == 13); + CHECK(ecount2 == 14); secp256k1_context_set_illegal_callback(vrfy, NULL, NULL); secp256k1_context_set_illegal_callback(sign, NULL, NULL); @@ -1879,9 +1891,9 @@ void test_ge(void) { * * When the endomorphism code is compiled in, p5 = lambda*p1 and p6 = lambda^2*p1 are added as well. */ - secp256k1_ge *ge = (secp256k1_ge *)malloc(sizeof(secp256k1_ge) * (1 + 4 * runs)); - secp256k1_gej *gej = (secp256k1_gej *)malloc(sizeof(secp256k1_gej) * (1 + 4 * runs)); - secp256k1_fe *zinv = (secp256k1_fe *)malloc(sizeof(secp256k1_fe) * (1 + 4 * runs)); + secp256k1_ge *ge = (secp256k1_ge *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_ge) * (1 + 4 * runs)); + secp256k1_gej *gej = (secp256k1_gej *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_gej) * (1 + 4 * runs)); + secp256k1_fe *zinv = (secp256k1_fe *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_fe) * (1 + 4 * runs)); secp256k1_fe zf; secp256k1_fe zfi2, zfi3; @@ -1919,7 +1931,7 @@ void test_ge(void) { /* Compute z inverses. */ { - secp256k1_fe *zs = malloc(sizeof(secp256k1_fe) * (1 + 4 * runs)); + secp256k1_fe *zs = checked_malloc(&ctx->error_callback, sizeof(secp256k1_fe) * (1 + 4 * runs)); for (i = 0; i < 4 * runs + 1; i++) { if (i == 0) { /* The point at infinity does not have a meaningful z inverse. Any should do. */ @@ -2020,7 +2032,7 @@ void test_ge(void) { /* Test adding all points together in random order equals infinity. */ { secp256k1_gej sum = SECP256K1_GEJ_CONST_INFINITY; - secp256k1_gej *gej_shuffled = (secp256k1_gej *)malloc((4 * runs + 1) * sizeof(secp256k1_gej)); + secp256k1_gej *gej_shuffled = (secp256k1_gej *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_gej)); for (i = 0; i < 4 * runs + 1; i++) { gej_shuffled[i] = gej[i]; } @@ -2041,9 +2053,9 @@ void test_ge(void) { /* Test batch gej -> ge conversion with and without known z ratios. */ { - secp256k1_fe *zr = (secp256k1_fe *)malloc((4 * runs + 1) * sizeof(secp256k1_fe)); - secp256k1_ge *ge_set_table = (secp256k1_ge *)malloc((4 * runs + 1) * sizeof(secp256k1_ge)); - secp256k1_ge *ge_set_all = (secp256k1_ge *)malloc((4 * runs + 1) * sizeof(secp256k1_ge)); + secp256k1_fe *zr = (secp256k1_fe *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_fe)); + secp256k1_ge *ge_set_table = (secp256k1_ge *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_ge)); + secp256k1_ge *ge_set_all = (secp256k1_ge *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_ge)); for (i = 0; i < 4 * runs + 1; i++) { /* Compute gej[i + 1].z / gez[i].z (with gej[n].z taken to be 1). */ if (i < 4 * runs) { @@ -3436,6 +3448,7 @@ void test_ecdsa_end_to_end(void) { unsigned char pubkeyc[65]; size_t pubkeyclen = 65; secp256k1_pubkey pubkey; + secp256k1_pubkey pubkey_tmp; unsigned char seckey[300]; size_t seckeylen = 300; @@ -3457,6 +3470,13 @@ void test_ecdsa_end_to_end(void) { memset(&pubkey, 0, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeyc, pubkeyclen) == 1); + /* Verify negation changes the key and changes it back */ + memcpy(&pubkey_tmp, &pubkey, sizeof(pubkey)); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey_tmp) == 1); + CHECK(memcmp(&pubkey_tmp, &pubkey, sizeof(pubkey)) != 0); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey_tmp) == 1); + CHECK(memcmp(&pubkey_tmp, &pubkey, sizeof(pubkey)) == 0); + /* Verify private key import and export. */ CHECK(ec_privkey_export_der(ctx, seckey, &seckeylen, privkey, secp256k1_rand_bits(1) == 1)); CHECK(ec_privkey_import_der(ctx, privkey2, seckey, seckeylen) == 1); @@ -4383,10 +4403,6 @@ void run_ecdsa_openssl(void) { # include "modules/ecdh/tests_impl.h" #endif -#ifdef ENABLE_MODULE_SCHNORR -# include "modules/schnorr/tests_impl.h" -#endif - #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/tests_impl.h" #endif @@ -4504,11 +4520,6 @@ int main(int argc, char **argv) { run_ecdsa_openssl(); #endif -#ifdef ENABLE_MODULE_SCHNORR - /* Schnorr tests */ - run_schnorr_tests(); -#endif - #ifdef ENABLE_MODULE_RECOVERY /* ECDSA pubkey recovery tests */ run_recovery_tests(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index bda6ee475c..b040bb0733 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -26,6 +26,11 @@ #include "secp256k1.c" #include "testrand_impl.h" +#ifdef ENABLE_MODULE_RECOVERY +#include "src/modules/recovery/main_impl.h" +#include "include/secp256k1_recovery.h" +#endif + /** stolen from tests.c */ void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { CHECK(a->infinity == b->infinity); @@ -77,7 +82,7 @@ int secp256k1_nonce_function_smallint(unsigned char *nonce32, const unsigned cha * function with an increased `attempt`. So if attempt > 0 this means we * need to change the nonce to avoid an infinite loop. */ if (attempt > 0) { - (*idata)++; + *idata = (*idata + 1) % EXHAUSTIVE_TEST_ORDER; } secp256k1_scalar_set_int(&s, *idata); secp256k1_scalar_get_b32(nonce32, &s); @@ -244,6 +249,7 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou for (i = 1; i < order; i++) { /* message */ for (j = 1; j < order; j++) { /* key */ for (k = 1; k < order; k++) { /* nonce */ + const int starting_k = k; secp256k1_ecdsa_signature sig; secp256k1_scalar sk, msg, r, s, expected_r; unsigned char sk32[32], msg32[32]; @@ -262,6 +268,11 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou CHECK(r == expected_r); CHECK((k * s) % order == (i + r * j) % order || (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + + /* Overflow means we've tried every possible nonce */ + if (k < starting_k) { + break; + } } } } @@ -276,6 +287,130 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou */ } +#ifdef ENABLE_MODULE_RECOVERY +void test_exhaustive_recovery_sign(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int i, j, k; + + /* Loop */ + for (i = 1; i < order; i++) { /* message */ + for (j = 1; j < order; j++) { /* key */ + for (k = 1; k < order; k++) { /* nonce */ + const int starting_k = k; + secp256k1_fe r_dot_y_normalized; + secp256k1_ecdsa_recoverable_signature rsig; + secp256k1_ecdsa_signature sig; + secp256k1_scalar sk, msg, r, s, expected_r; + unsigned char sk32[32], msg32[32]; + int expected_recid; + int recid; + secp256k1_scalar_set_int(&msg, i); + secp256k1_scalar_set_int(&sk, j); + secp256k1_scalar_get_b32(sk32, &sk); + secp256k1_scalar_get_b32(msg32, &msg); + + secp256k1_ecdsa_sign_recoverable(ctx, &rsig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + + /* Check directly */ + secp256k1_ecdsa_recoverable_signature_load(ctx, &r, &s, &recid, &rsig); + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + /* In computing the recid, there is an overflow condition that is disabled in + * scalar_low_impl.h `secp256k1_scalar_set_b32` because almost every r.y value + * will exceed the group order, and our signing code always holds out for r + * values that don't overflow, so with a proper overflow check the tests would + * loop indefinitely. */ + r_dot_y_normalized = group[k].y; + secp256k1_fe_normalize(&r_dot_y_normalized); + /* Also the recovery id is flipped depending if we hit the low-s branch */ + if ((k * s) % order == (i + r * j) % order) { + expected_recid = secp256k1_fe_is_odd(&r_dot_y_normalized) ? 1 : 0; + } else { + expected_recid = secp256k1_fe_is_odd(&r_dot_y_normalized) ? 0 : 1; + } + CHECK(recid == expected_recid); + + /* Convert to a standard sig then check */ + secp256k1_ecdsa_recoverable_signature_convert(ctx, &sig, &rsig); + secp256k1_ecdsa_signature_load(ctx, &r, &s, &sig); + /* Note that we compute expected_r *after* signing -- this is important + * because our nonce-computing function function might change k during + * signing. */ + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + + /* Overflow means we've tried every possible nonce */ + if (k < starting_k) { + break; + } + } + } + } +} + +void test_exhaustive_recovery_verify(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + /* This is essentially a copy of test_exhaustive_verify, with recovery added */ + int s, r, msg, key; + for (s = 1; s < order; s++) { + for (r = 1; r < order; r++) { + for (msg = 1; msg < order; msg++) { + for (key = 1; key < order; key++) { + secp256k1_ge nonconst_ge; + secp256k1_ecdsa_recoverable_signature rsig; + secp256k1_ecdsa_signature sig; + secp256k1_pubkey pk; + secp256k1_scalar sk_s, msg_s, r_s, s_s; + secp256k1_scalar s_times_k_s, msg_plus_r_times_sk_s; + int recid = 0; + int k, should_verify; + unsigned char msg32[32]; + + secp256k1_scalar_set_int(&s_s, s); + secp256k1_scalar_set_int(&r_s, r); + secp256k1_scalar_set_int(&msg_s, msg); + secp256k1_scalar_set_int(&sk_s, key); + secp256k1_scalar_get_b32(msg32, &msg_s); + + /* Verify by hand */ + /* Run through every k value that gives us this r and check that *one* works. + * Note there could be none, there could be multiple, ECDSA is weird. */ + should_verify = 0; + for (k = 0; k < order; k++) { + secp256k1_scalar check_x_s; + r_from_k(&check_x_s, group, k); + if (r_s == check_x_s) { + secp256k1_scalar_set_int(&s_times_k_s, k); + secp256k1_scalar_mul(&s_times_k_s, &s_times_k_s, &s_s); + secp256k1_scalar_mul(&msg_plus_r_times_sk_s, &r_s, &sk_s); + secp256k1_scalar_add(&msg_plus_r_times_sk_s, &msg_plus_r_times_sk_s, &msg_s); + should_verify |= secp256k1_scalar_eq(&s_times_k_s, &msg_plus_r_times_sk_s); + } + } + /* nb we have a "high s" rule */ + should_verify &= !secp256k1_scalar_is_high(&s_s); + + /* We would like to try recovering the pubkey and checking that it matches, + * but pubkey recovery is impossible in the exhaustive tests (the reason + * being that there are 12 nonzero r values, 12 nonzero points, and no + * overlap between the sets, so there are no valid signatures). */ + + /* Verify by converting to a standard signature and calling verify */ + secp256k1_ecdsa_recoverable_signature_save(&rsig, &r_s, &s_s, recid); + secp256k1_ecdsa_recoverable_signature_convert(ctx, &sig, &rsig); + memcpy(&nonconst_ge, &group[sk_s], sizeof(nonconst_ge)); + secp256k1_pubkey_save(&pk, &nonconst_ge); + CHECK(should_verify == + secp256k1_ecdsa_verify(ctx, &sig, msg32, &pk)); + } + } + } + } +} +#endif + int main(void) { int i; secp256k1_gej groupj[EXHAUSTIVE_TEST_ORDER]; @@ -324,6 +459,12 @@ int main(void) { test_exhaustive_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); test_exhaustive_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); +#ifdef ENABLE_MODULE_RECOVERY + test_exhaustive_recovery_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_recovery_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); +#endif + + secp256k1_context_destroy(ctx); return 0; } diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 4eef4ded47..4092a86c91 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -57,7 +57,10 @@ static SECP256K1_INLINE void secp256k1_callback_call(const secp256k1_callback * #endif /* Like assert(), but when VERIFY is defined, and side-effect safe. */ -#ifdef VERIFY +#if defined(COVERAGE) +#define VERIFY_CHECK(check) +#define VERIFY_SETUP(stmt) +#elif defined(VERIFY) #define VERIFY_CHECK CHECK #define VERIFY_SETUP(stmt) do { stmt; } while(0) #else diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index c62e6ae838..e6b45a3b5e 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) connman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, "", true); + CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); dummyNode2.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode2, *connman); dummyNode2.nVersion = 1; @@ -83,7 +83,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) connman->ClearBanned(); ForceSetArg("-banscore", "111"); // because 11 is my favorite number CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; @@ -109,7 +109,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, "", true); + CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", true); dummyNode.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode, *connman); dummyNode.nVersion = 1; diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 31ed1a50b9..0ed71b96fc 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -17,35 +17,44 @@ #include <boost/test/unit_test.hpp> -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace { +//! equality test +bool operator==(const Coin &a, const Coin &b) { + // Empty Coin objects are always equal. + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && + a.nHeight == b.nHeight && + a.out == b.out; +} + class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map<uint256, CCoins> map_; + std::map<COutPoint, Coin> map_; public: - bool GetCoins(const uint256& txid, CCoins& coins) const + bool GetCoin(const COutPoint& outpoint, Coin& coin) const { - std::map<uint256, CCoins>::const_iterator it = map_.find(txid); + std::map<COutPoint, Coin>::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && insecure_rand() % 2 == 0) { + coin = it->second; + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256& txid) const + bool HaveCoin(const COutPoint& outpoint) const { - CCoins coins; - return GetCoins(txid, coins); + Coin coin; + return GetCoin(outpoint, coin); } uint256 GetBestBlock() const { return hashBestBlock_; } @@ -55,8 +64,8 @@ public: for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -78,9 +87,12 @@ public: { // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); + size_t count = 0; for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); + ++count; } + BOOST_CHECK_EQUAL(GetCacheSize(), count); BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -97,7 +109,7 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // This is a large randomized insert/remove simulation test on a variable-size // stack of caches on top of CCoinsViewTest. // -// It will randomly create/update/delete CCoins entries to a tip of caches, with +// It will randomly create/update/delete Coin entries to a tip of caches, with // txids picked from a limited list of random 256-bit hashes. Occasionally, a // new tip is added to the stack of caches, or the tip is flushed and removed. // @@ -109,13 +121,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) bool removed_all_caches = false; bool reached_4_caches = false; bool added_an_entry = false; + bool added_an_unspendable_entry = false; bool removed_an_entry = false; bool updated_an_entry = false; bool found_an_entry = false; bool missed_an_entry = false; + bool uncached_an_entry = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -133,36 +147,51 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Do a random modification. { uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. - CCoins& coins = result[txid]; - CCoinsModifier entry = stack.back()->ModifyCoins(txid); - BOOST_CHECK(coins == *entry); - if (insecure_rand() % 5 == 0 || coins.IsPruned()) { - if (coins.IsPruned()) { - added_an_entry = true; + Coin& coin = result[COutPoint(txid, 0)]; + const Coin& entry = (insecure_rand() % 500 == 0) ? AccessByTxid(*stack.back(), txid) : stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK(coin == entry); + + if (insecure_rand() % 5 == 0 || coin.IsSpent()) { + Coin newcoin; + newcoin.out.nValue = insecure_rand(); + newcoin.nHeight = 1; + if (insecure_rand() % 16 == 0 && coin.IsSpent()) { + newcoin.out.scriptPubKey.assign(1 + (insecure_rand() & 0x3F), OP_RETURN); + BOOST_CHECK(newcoin.out.scriptPubKey.IsUnspendable()); + added_an_unspendable_entry = true; } else { - updated_an_entry = true; + newcoin.out.scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting + (coin.IsSpent() ? added_an_entry : updated_an_entry) = true; + coin = newcoin; } - coins.nVersion = insecure_rand(); - coins.vout.resize(1); - coins.vout[0].nValue = insecure_rand(); - *entry = coins; + stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), !coin.IsSpent() || insecure_rand() & 1); } else { - coins.Clear(); - entry->Clear(); removed_an_entry = true; + coin.Clear(); + stack.back()->SpendCoin(COutPoint(txid, 0)); } } + // One every 10 iterations, remove a random entry from the cache + if (insecure_rand() % 10 == 0) { + COutPoint out(txids[insecure_rand() % txids.size()], 0); + int cacheid = insecure_rand() % stack.size(); + stack[cacheid]->Uncache(out); + uncached_an_entry |= !stack[cacheid]->HaveCoinInCache(out); + } + // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - found_an_entry = true; - } else { - BOOST_CHECK(it->second.IsPruned()); + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); + if (coin.IsSpent()) { missed_an_entry = true; + } else { + BOOST_CHECK(stack.back()->HaveCoinInCache(it->first)); + found_an_entry = true; } } BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { @@ -211,25 +240,27 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(removed_all_caches); BOOST_CHECK(reached_4_caches); BOOST_CHECK(added_an_entry); + BOOST_CHECK(added_an_unspendable_entry); BOOST_CHECK(removed_an_entry); BOOST_CHECK(updated_an_entry); BOOST_CHECK(found_an_entry); BOOST_CHECK(missed_an_entry); + BOOST_CHECK(uncached_an_entry); } -typedef std::tuple<CTransaction,CTxUndo,CCoins> TxData; // Store of all necessary tx and undo data for next test -std::map<uint256, TxData> alltxs; - -TxData &FindRandomFrom(const std::set<uint256> &txidset) { - assert(txidset.size()); - std::set<uint256>::iterator txIt = txidset.lower_bound(GetRandHash()); - if (txIt == txidset.end()) { - txIt = txidset.begin(); +typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; +UtxoData utxoData; + +UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { + assert(utxoSet.size()); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(GetRandHash(), 0)); + if (utxoSetIt == utxoSet.end()) { + utxoSetIt = utxoSet.begin(); } - std::map<uint256, TxData>::iterator txdit = alltxs.find(*txIt); - assert(txdit != alltxs.end()); - return txdit->second; + auto utxoDataIt = utxoData.find(*utxoSetIt); + assert(utxoDataIt != utxoData.end()); + return utxoDataIt; } @@ -242,7 +273,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { bool spent_a_duplicate_coinbase = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -250,10 +281,10 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Track the txids we've used in various sets - std::set<uint256> coinbaseids; - std::set<uint256> disconnectedids; - std::set<uint256> duplicateids; - std::set<uint256> utxoset; + std::set<COutPoint> coinbase_coins; + std::set<COutPoint> disconnected_coins; + std::set<COutPoint> duplicate_coins; + std::set<COutPoint> utxoset; for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { uint32_t randiter = insecure_rand(); @@ -264,23 +295,24 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vin.resize(1); tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate + tx.vout[0].scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting unsigned int height = insecure_rand(); - CCoins oldcoins; + Coin old_coin; // 2/20 times create a new coinbase - if (randiter % 20 < 2 || coinbaseids.size() < 10) { + if (randiter % 20 < 2 || coinbase_coins.size() < 10) { // 1/10 of those times create a duplicate coinbase - if (insecure_rand() % 10 == 0 && coinbaseids.size()) { - TxData &txd = FindRandomFrom(coinbaseids); + if (insecure_rand() % 10 == 0 && coinbase_coins.size()) { + auto utxod = FindRandomFrom(coinbase_coins); // Reuse the exact same coinbase - tx = std::get<0>(txd); + tx = std::get<0>(utxod->second); // shouldn't be available for reconnection if its been duplicated - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); - duplicateids.insert(tx.GetHash()); + duplicate_coins.insert(utxod->first); } else { - coinbaseids.insert(tx.GetHash()); + coinbase_coins.insert(COutPoint(tx.GetHash(), 0)); } assert(CTransaction(tx).IsCoinBase()); } @@ -288,114 +320,118 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // 17/20 times reconnect previous or add a regular tx else { - uint256 prevouthash; + COutPoint prevout; // 1/20 times reconnect a previously disconnected tx - if (randiter % 20 == 2 && disconnectedids.size()) { - TxData &txd = FindRandomFrom(disconnectedids); - tx = std::get<0>(txd); - prevouthash = tx.vin[0].prevout.hash; - if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevouthash)) { - disconnectedids.erase(tx.GetHash()); + if (randiter % 20 == 2 && disconnected_coins.size()) { + auto utxod = FindRandomFrom(disconnected_coins); + tx = std::get<0>(utxod->second); + prevout = tx.vin[0].prevout; + if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevout)) { + disconnected_coins.erase(utxod->first); continue; } // If this tx is already IN the UTXO, then it must be a coinbase, and it must be a duplicate - if (utxoset.count(tx.GetHash())) { + if (utxoset.count(utxod->first)) { assert(CTransaction(tx).IsCoinBase()); - assert(duplicateids.count(tx.GetHash())); + assert(duplicate_coins.count(utxod->first)); } - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); } // 16/20 times create a regular tx else { - TxData &txd = FindRandomFrom(utxoset); - prevouthash = std::get<0>(txd).GetHash(); + auto utxod = FindRandomFrom(utxoset); + prevout = utxod->first; // Construct the tx to spend the coins of prevouthash - tx.vin[0].prevout.hash = prevouthash; - tx.vin[0].prevout.n = 0; + tx.vin[0].prevout = prevout; assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevouthash]; + old_coin = result[prevout]; // Update the expected result of prevouthash to know these coins are spent - result[prevouthash].Clear(); + result[prevout].Clear(); - utxoset.erase(prevouthash); + utxoset.erase(prevout); // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) + if (duplicate_coins.count(prevout)) { spent_a_duplicate_coinbase = true; + } } // Update the expected result to know about the new output coins - result[tx.GetHash()].FromTx(tx, height); + assert(tx.vout.size() == 1); + const COutPoint outpoint(tx.GetHash(), 0); + result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); // Call UpdateCoins on the top cache CTxUndo undo; UpdateCoins(tx, *(stack.back()), undo, height); // Update the utxo set for future spends - utxoset.insert(tx.GetHash()); + utxoset.insert(outpoint); // Track this tx and undo info to use later - alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); - } - - //1/20 times undo a previous transaction - else if (utxoset.size()) { - TxData &txd = FindRandomFrom(utxoset); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,old_coin)); + } else if (utxoset.size()) { + //1/20 times undo a previous transaction + auto utxod = FindRandomFrom(utxoset); - CTransaction &tx = std::get<0>(txd); - CTxUndo &undo = std::get<1>(txd); - CCoins &origcoins = std::get<2>(txd); - - uint256 undohash = tx.GetHash(); + CTransaction &tx = std::get<0>(utxod->second); + CTxUndo &undo = std::get<1>(utxod->second); + Coin &orig_coin = std::get<2>(utxod->second); // Update the expected result // Remove new outputs - result[undohash].Clear(); + result[utxod->first].Clear(); // If not coinbase restore prevout if (!tx.IsCoinBase()) { - result[tx.vin[0].prevout.hash] = origcoins; + result[tx.vin[0].prevout] = orig_coin; } // Disconnect the tx from the current UTXO // See code in DisconnectBlock // remove outputs - { - CCoinsModifier outs = stack.back()->ModifyCoins(undohash); - outs->Clear(); - } + stack.back()->SpendCoin(utxod->first); // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const CTxInUndo &undoin = undo.vprevout[0]; - ApplyTxInUndo(undoin, *(stack.back()), out); + Coin coin = undo.vprevout[0]; + ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection - disconnectedids.insert(undohash); + disconnected_coins.insert(utxod->first); // Update the utxoset - utxoset.erase(undohash); + utxoset.erase(utxod->first); if (!tx.IsCoinBase()) - utxoset.insert(tx.vin[0].prevout.hash); + utxoset.insert(tx.vin[0].prevout); } // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - } else { - BOOST_CHECK(it->second.IsPruned()); - } + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); } } + // One every 10 iterations, remove a random entry from the cache + if (utxoset.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(utxoset)->first); + } + if (disconnected_coins.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(disconnected_coins)->first); + } + if (duplicate_coins.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(duplicate_coins)->first); + } + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && insecure_rand() % 2 == 0) { @@ -433,53 +469,36 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); - CCoins cc1; + CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + Coin cc1; ss1 >> cc1; - BOOST_CHECK_EQUAL(cc1.nVersion, 1); BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); - BOOST_CHECK_EQUAL(cc1.vout.size(), 2); - BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); - BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); - BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); - BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + BOOST_CHECK_EQUAL(cc1.out.nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); - CCoins cc2; + CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + Coin cc2; ss2 >> cc2; - BOOST_CHECK_EQUAL(cc2.nVersion, 1); BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); - BOOST_CHECK_EQUAL(cc2.vout.size(), 17); - for (int i = 0; i < 17; i++) { - BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); - } - BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); - BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + BOOST_CHECK_EQUAL(cc2.out.nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ssx(SER_DISK, CLIENT_VERSION); - BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); - - CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); - CCoins cc3; + CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + Coin cc3; ss3 >> cc3; - BOOST_CHECK_EQUAL(cc3.nVersion, 0); BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); - BOOST_CHECK_EQUAL(cc3.vout.size(), 1); - BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); - BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); - BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + BOOST_CHECK_EQUAL(cc3.out.nValue, 0); + BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); try { - CCoins cc4; + Coin cc4; ss4 >> cc4; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { @@ -490,16 +509,16 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); - CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { - CCoins cc5; + Coin cc5; ss5 >> cc5; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { } } -const static uint256 TXID; +const static COutPoint OUTPOINT; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -514,15 +533,15 @@ const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; const static auto CLEAN_FLAGS = {char(0), FRESH}; const static auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins& coins) +void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsSpent()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + coin.out.nValue = value; + coin.nHeight = 1; + assert(!coin.IsSpent()); } } @@ -535,25 +554,23 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + SetCoinsValue(value, entry.coin); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coin.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); @@ -581,10 +598,10 @@ public: CCoinsViewCacheTest cache{&base}; }; -void CheckAccessCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoins(TXID); + test.cache.AccessCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -603,39 +620,39 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Base Cache Result Cache Result * Value Value Value Flags Flags */ - CheckAccessCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); } -void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - SetCoinsValue(modify_value, *test.cache.ModifyCoins(TXID)); + test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -645,79 +662,55 @@ void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_va BOOST_CHECK_EQUAL(result_flags, expected_flags); }; -BOOST_AUTO_TEST_CASE(ccoins_modify) +BOOST_AUTO_TEST_CASE(ccoins_spend) { - /* Check ModifyCoin behavior, requesting a coin from a cache view layered on - * top of a base view, writing a modification to the coin, and then checking + /* Check SpendCoin behavior, requesting a coin from a cache view layered on + * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. * - * Base Cache Write Result Cache Result - * Value Value Value Value Flags Flags + * Base Cache Result Cache Result + * Value Value Value Flags Flags */ - CheckModifyCoins(ABSENT, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(ABSENT, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(PRUNED, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); + CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } -void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); CAmount result_value; char result_flags; try { - SetCoinsValue(modify_value, *test.cache.ModifyNewCoins(TXID, coinbase)); + CTxOut output; + output.nValue = modify_value; + test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + test.cache.SelfTest(); GetCoinsMapEntry(test.cache.map(), result_value, result_flags); } catch (std::logic_error& e) { result_value = FAIL; @@ -728,64 +721,46 @@ void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount mo BOOST_CHECK_EQUAL(result_flags, expected_flags); } -// Simple wrapper for CheckModifyNewCoinsBase function above that loops through +// Simple wrapper for CheckAddCoinBase function above that loops through // different possible base_values, making sure each one gives the same results. -// This wrapper lets the modify_new test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::ModifyNewCoins implementation +// This wrapper lets the coins_add test below be shorter and less repetitive, +// while still verifying that the CoinsViewCache::AddCoin implementation // ignores base values. template <typename... Args> -void CheckModifyNewCoins(Args&&... args) +void CheckAddCoin(Args&&... args) { for (CAmount base_value : {ABSENT, PRUNED, VALUE1}) - CheckModifyNewCoinsBase(base_value, std::forward<Args>(args)...); + CheckAddCoinBase(base_value, std::forward<Args>(args)...); } -BOOST_AUTO_TEST_CASE(ccoins_modify_new) +BOOST_AUTO_TEST_CASE(ccoins_add) { - /* Check ModifyNewCoin behavior, requesting a new coin from a cache view, + /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the ModifyNewCoin coinbase argument set to false, and to true. + * with the AddCoin potential_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result Coinbase - * Value Value Value Flags Flags + * Cache Write Result Cache Result potential_overwrite + * Value Value Value Flags Flags */ - CheckModifyNewCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY , false); - CheckModifyNewCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, 0 , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); } void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d8de765db1..bb7e473248 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -128,6 +128,23 @@ BOOST_AUTO_TEST_CASE(siphash) tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); + + // Check consistency between CSipHasher and SipHashUint256[Extra]. + FastRandomContext ctx; + for (int i = 0; i < 16; ++i) { + uint64_t k1 = ctx.rand64(); + uint64_t k2 = ctx.rand64(); + uint256 x = GetRandHash(); + uint32_t n = ctx.rand32(); + uint8_t nb[4]; + WriteLE32(nb, n); + CSipHasher sip256(k1, k2); + sip256.Write(x.begin(), 32); + CSipHasher sip288 = sip256; + sip288.Write(nb, 4); + BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); + BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 0c7f3e5e23..66354699b2 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -175,12 +175,12 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) bool fInboundIn = false; // Test that fFeeler is false by default. - std::unique_ptr<CNode> pnode1(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, pszDest, fInboundIn)); + std::unique_ptr<CNode> pnode1(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, CAddress(), pszDest, fInboundIn)); BOOST_CHECK(pnode1->fInbound == false); BOOST_CHECK(pnode1->fFeeler == false); fInboundIn = true; - std::unique_ptr<CNode> pnode2(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, pszDest, fInboundIn)); + std::unique_ptr<CNode> pnode2(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, CAddress(), pszDest, fInboundIn)); BOOST_CHECK(pnode2->fInbound == true); BOOST_CHECK(pnode2->fFeeler == false); } diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index ede68f23d7..0789b2e80c 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -112,7 +112,8 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(CCoins(txFrom, 0), txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + const CTxOut& output = txFrom.vout[txTo[i].vin[0].prevout.n]; + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else @@ -316,7 +317,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops)); txFrom.vout[6].nValue = 6000; - coins.ModifyCoins(txFrom.GetHash())->FromTx(txFrom, 0); + AddCoins(coins, txFrom, 0); CMutableTransaction txTo; txTo.vout.resize(1); diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 92781d763d..4e117448fe 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -102,7 +102,7 @@ void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableT spendingTx.vout[0].nValue = 1; spendingTx.vout[0].scriptPubKey = CScript(); - coins.ModifyCoins(creationTx.GetHash())->FromTx(creationTx, 0); + AddCoins(coins, creationTx, 0); } BOOST_AUTO_TEST_CASE(GetTxSigOpCost) diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index c4983f6f5c..de14251601 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -59,9 +59,8 @@ bool read_stdin(std::vector<char> &data) { return length==0; } -int main(int argc, char **argv) +int do_fuzz() { - ECCVerifyHandle globalVerifyHandle; std::vector<char> buffer; if (!read_stdin(buffer)) return 0; @@ -169,8 +168,8 @@ int main(int argc, char **argv) { try { - CCoins block; - ds >> block; + Coin coin; + ds >> coin; } catch (const std::ios_base::failure& e) {return 0;} break; } @@ -256,3 +255,23 @@ int main(int argc, char **argv) return 0; } +int main(int argc, char **argv) +{ + ECCVerifyHandle globalVerifyHandle; +#ifdef __AFL_INIT + // Enable AFL deferred forkserver mode. Requires compilation using + // afl-clang-fast++. See fuzzing.md for details. + __AFL_INIT(); +#endif + +#ifdef __AFL_LOOP + // Enable AFL persistent mode. Requires compilation using afl-clang-fast++. + // See fuzzing.md for details. + while (__AFL_LOOP(1000)) { + do_fuzz(); + } + return 0; +#else + return do_fuzz(); +#endif +} diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 67610301d7..5c7516fbf1 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -307,14 +307,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50*CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21*CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22*CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } @@ -470,19 +470,20 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for (int i=0; i<20; i++) threadGroup.create_thread(boost::bind(&CCheckQueue<CScriptCheck>::Thread, boost::ref(scriptcheckqueue))); - CCoins coins; - coins.nVersion = 1; - coins.fCoinBase = false; + std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { - CTxOut txout; - txout.nValue = 1000; - txout.scriptPubKey = scriptPubKey; - coins.vout.push_back(txout); + Coin coin; + coin.nHeight = 1; + coin.fCoinBase = false; + coin.out.nValue = 1000; + coin.out.scriptPubKey = scriptPubKey; + coins.emplace_back(std::move(coin)); } for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector<CScriptCheck> vChecks; - CScriptCheck check(coins, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); + const CTxOut& output = coins[tx.vin[i].prevout.n].out; + CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); control.Add(vChecks); diff --git a/src/txdb.cpp b/src/txdb.cpp index 42dc31760b..c8f5090293 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include <boost/thread.hpp> +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,17 +25,40 @@ static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinEntry { + COutPoint* outpoint; + char key; + CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {} + + template<typename Stream> + void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template<typename Stream> + void Unserialize(Stream& s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; + +} CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -50,10 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) + batch.Erase(entry); else - batch.Write(std::make_pair(DB_COINS, it->first), it->second.coins); + batch.Write(entry, it->second.coin); changed++; } count++; @@ -63,8 +88,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint(BCLog::COINDB, "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); - return db.WriteBatch(batch); + bool ret = db.WriteBatch(batch); + LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); + return ret; +} + +size_t CCoinsViewDB::EstimateSize() const +{ + return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -96,25 +127,31 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (i->pcursor->Valid()) { + CoinEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; + } else { + i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false + } return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { - return pcursor->GetValue(coins); + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const @@ -124,14 +161,18 @@ unsigned int CCoinsViewDBCursor::GetValueSize() const bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { @@ -211,3 +252,103 @@ bool CBlockTreeDB::LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&) return true; } + +namespace { + +//! Legacy class to deserialize pre-pertxout database entries without reindex. +class CCoins +{ +public: + //! whether transaction is a coinbase + bool fCoinBase; + + //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped + std::vector<CTxOut> vout; + + //! at which height this transaction was included in the active block chain + int nHeight; + + //! empty constructor + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } + + template<typename Stream> + void Unserialize(Stream &s) { + unsigned int nCode = 0; + // version + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + // header code + ::Unserialize(s, VARINT(nCode)); + fCoinBase = nCode & 1; + std::vector<bool> vAvail(2, false); + vAvail[0] = (nCode & 2) != 0; + vAvail[1] = (nCode & 4) != 0; + unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); + // spentness bitmask + while (nMaskCode > 0) { + unsigned char chAvail = 0; + ::Unserialize(s, chAvail); + for (unsigned int p = 0; p < 8; p++) { + bool f = (chAvail & (1 << p)) != 0; + vAvail.push_back(f); + } + if (chAvail != 0) + nMaskCode--; + } + // txouts themself + vout.assign(vAvail.size(), CTxOut()); + for (unsigned int i = 0; i < vAvail.size(); i++) { + if (vAvail[i]) + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); + } + // coinbase height + ::Unserialize(s, VARINT(nHeight)); + } +}; + +} + +/** Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr<CDBIterator> pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair<unsigned char, uint256> key; + if (pcursor->GetKey(key) && key.first == DB_COINS) { + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + pcursor->Next(); + } else { + break; + } + } + db.WriteBatch(batch); + return true; +} diff --git a/src/txdb.h b/src/txdb.h index 117e7201fb..974dd4ebe3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -22,9 +22,7 @@ class uint256; //! Compensate for extra memory peak (x1.5-x1.9) at flush time. static constexpr int DB_PEAK_USAGE_FACTOR = 2; //! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; -//! Always periodic flush if less than this much space still available. -static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; +static constexpr int MAX_BLOCK_COINSDB_USAGE = 10 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) @@ -73,11 +71,15 @@ protected: public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + + //! Attempt to update from an older database format. Returns whether an error occurred. + bool Upgrade(); + size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ @@ -86,8 +88,8 @@ class CCoinsViewDBCursor: public CCoinsViewCursor public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -97,7 +99,7 @@ private: CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr<CDBIterator> pcursor; - std::pair<char, uint256> keyTmp; + std::pair<char, COutPoint> keyTmp; friend class CCoinsViewDB; }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 33df0536d0..17389db9f0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -24,7 +24,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); - nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); + nUsageSize = RecursiveDynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); @@ -343,17 +343,10 @@ CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : nCheckFrequency = 0; } -void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); - - auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); - - // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first->hash == hashTx) { - coins.Spend(it->first->n); // and remove those outputs from coins - it++; - } + return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const @@ -531,9 +524,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); - if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { + const Coin &coin = pcoins->AccessCoin(txin.prevout); + if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; } @@ -661,8 +654,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); - assert(coins && coins->IsAvailable(txin.prevout.n)); + assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -898,20 +890,24 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } else { + return false; + } } - return (base->GetCoins(txid, coins) && !coins.IsPruned()); + return (base->GetCoin(outpoint, coin) && !coin.IsSpent()); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins(txid); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1022,7 +1018,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { } } -void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining) { +void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; @@ -1053,11 +1049,10 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (exists(txin.prevout.hash)) - continue; - auto iter = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || iter->first->hash != txin.prevout.hash) - pvNoSpendsRemaining->push_back(txin.prevout.hash); + if (exists(txin.prevout.hash)) continue; + if (!mapNextTx.count(txin.prevout)) { + pvNoSpendsRemaining->push_back(txin.prevout); + } } } } @@ -1074,3 +1069,5 @@ bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLi return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index a91eb5be54..0316b42ba2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -24,14 +24,15 @@ #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" #include "boost/multi_index/hashed_index.hpp" +#include <boost/multi_index/sequenced_index.hpp> #include <boost/signals2/signal.hpp> class CAutoFile; class CBlockIndex; -/** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ -static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; +/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ +static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { @@ -185,7 +186,7 @@ private: const LockPoints& lp; }; -// extracts a TxMemPoolEntry's transaction hash +// extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -193,6 +194,11 @@ struct mempoolentry_txid { return entry.GetTx().GetHash(); } + + result_type operator() (const CTransactionRef& tx) const + { + return tx->GetHash(); + } }; /** \class CompareTxMemPoolEntryByDescendantScore @@ -321,6 +327,20 @@ enum class MemPoolRemovalReason { REPLACED //! Removed for replacement }; +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -509,7 +529,7 @@ public: void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector<uint256>& vtxid); - void pruneSpent(const uint256& hash, CCoins &coins); + bool isSpent(const COutPoint& outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** @@ -570,10 +590,10 @@ public: CFeeRate GetMinFee(size_t sizelimit) const; /** Remove transactions from the mempool until its dynamic size is <= sizelimit. - * pvNoSpendsRemaining, if set, will be populated with the list of transactions + * pvNoSpendsRemaining, if set, will be populated with the list of outpoints * which are not in mempool which no longer have any spends in this mempool. */ - void TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining=NULL); + void TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining=NULL); /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); @@ -599,6 +619,13 @@ public: return (mapTx.count(hash) != 0); } + bool exists(const COutPoint& outpoint) const + { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); + } + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; @@ -658,8 +685,99 @@ protected: public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; +}; + +/** + * DisconnectedBlockTransactions + + * During the reorg, it's desirable to re-add previously confirmed transactions + * to the mempool, so that anything not re-confirmed in the new chain is + * available to be mined. However, it's more efficient to wait until the reorg + * is complete and process all still-unconfirmed transactions at that time, + * since we expect most confirmed transactions to (typically) still be + * confirmed in the new chain, and re-accepting to the memory pool is expensive + * (and therefore better to not do in the middle of reorg-processing). + * Instead, store the disconnected transactions (in order!) as we go, remove any + * that are included in blocks in the new chain, and then process the remaining + * still-unconfirmed transactions at the end. + */ + +// multi_index tag names +struct txid_index {}; +struct insertion_order {}; + +struct DisconnectedBlockTransactions { + typedef boost::multi_index_container< + CTransactionRef, + boost::multi_index::indexed_by< + // sorted by txid + boost::multi_index::hashed_unique< + boost::multi_index::tag<txid_index>, + mempoolentry_txid, + SaltedTxidHasher + >, + // sorted by order in the blockchain + boost::multi_index::sequenced< + boost::multi_index::tag<insertion_order> + > + > + > indexed_disconnected_transactions; + + // It's almost certainly a logic bug if we don't clear out queuedTx before + // destruction, as we add to it while disconnecting blocks, and then we + // need to re-process remaining transactions to ensure mempool consistency. + // For now, assert() that we've emptied out this object on destruction. + // This assert() can always be removed if the reorg-processing code were + // to be refactored such that this assumption is no longer true (for + // instance if there was some other way we cleaned up the mempool after a + // reorg, besides draining this object). + ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } + + indexed_disconnected_transactions queuedTx; + uint64_t cachedInnerUsage = 0; + + // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as + // no exact formula for boost::multi_index_contained is implemented. + size_t DynamicMemoryUsage() const { + return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void*)) * queuedTx.size() + cachedInnerUsage; + } + + void addTransaction(const CTransactionRef& tx) + { + queuedTx.insert(tx); + cachedInnerUsage += RecursiveDynamicUsage(tx); + } + + // Remove entries based on txid_index, and update memory usage. + void removeForBlock(const std::vector<CTransactionRef>& vtx) + { + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (auto const &tx : vtx) { + auto it = queuedTx.find(tx->GetHash()); + if (it != queuedTx.end()) { + cachedInnerUsage -= RecursiveDynamicUsage(*it); + queuedTx.erase(it); + } + } + } + + // Remove an entry by insertion_order index, and update memory usage. + void removeEntry(indexed_disconnected_transactions::index<insertion_order>::type::iterator entry) + { + cachedInnerUsage -= RecursiveDynamicUsage(*entry); + queuedTx.get<insertion_order>().erase(entry); + } + + void clear() + { + cachedInnerUsage = 0; + queuedTx.clear(); + } }; #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/undo.h b/src/undo.h index a94e31f5cc..3749d5d7a8 100644 --- a/src/undo.h +++ b/src/undo.h @@ -7,58 +7,90 @@ #define BITCOIN_UNDO_H #include "compressor.h" +#include "consensus/consensus.h" #include "primitives/transaction.h" #include "serialize.h" /** Undo information for a CTxIn * - * Contains the prevout's CTxOut being spent, and if this was the - * last output of the affected transaction, its metadata as well - * (coinbase or not, height, transaction version) + * Contains the prevout's CTxOut being spent, and its metadata as well + * (coinbase or not, height). The serialization contains a dummy value of + * zero. This is be compatible with older versions which expect to see + * the transaction version there. */ -class CTxInUndo +class TxInUndoSerializer { -public: - CTxOut txout; // the txout data before being spent - bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase - unsigned int nHeight; // if the outpoint was the last unspent: its height - int nVersion; // if the outpoint was the last unspent: its version - - CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } + const Coin* txout; +public: template<typename Stream> void Serialize(Stream &s) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion)); - ::Serialize(s, CTxOutCompressor(REF(txout))); + ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1 : 0))); + if (txout->nHeight > 0) { + // Required to maintain compatibility with older undo format. + ::Serialize(s, (unsigned char)0); + } + ::Serialize(s, CTxOutCompressor(REF(txout->out))); } + TxInUndoSerializer(const Coin* coin) : txout(coin) {} +}; + +class TxInUndoDeserializer +{ + Coin* txout; + +public: template<typename Stream> void Unserialize(Stream &s) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - nHeight = nCode / 2; - fCoinBase = nCode & 1; - if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion)); - ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); + txout->nHeight = nCode / 2; + txout->fCoinBase = nCode & 1; + if (txout->nHeight > 0) { + // Old versions stored the version number for the last spend of + // a transaction's outputs. Non-final spends were indicated with + // height = 0. + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + } + ::Unserialize(s, REF(CTxOutCompressor(REF(txout->out)))); } + + TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); + /** Undo information for a CTransaction */ class CTxUndo { public: // undo information for all txins - std::vector<CTxInUndo> vprevout; + std::vector<Coin> vprevout; - ADD_SERIALIZE_METHODS; + template <typename Stream> + void Serialize(Stream& s) const { + // TODO: avoid reimplementing vector serializer + uint64_t count = vprevout.size(); + ::Serialize(s, COMPACTSIZE(REF(count))); + for (const auto& prevout : vprevout) { + ::Serialize(s, REF(TxInUndoSerializer(&prevout))); + } + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vprevout); + template <typename Stream> + void Unserialize(Stream& s) { + // TODO: avoid reimplementing vector deserializer + uint64_t count = 0; + ::Unserialize(s, COMPACTSIZE(count)); + if (count > MAX_INPUTS_PER_BLOCK) { + throw std::ios_base::failure("Too many input undo records"); + } + vprevout.resize(count); + for (auto& prevout : vprevout) { + ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); + } } }; diff --git a/src/util.h b/src/util.h index 229478d835..4386ddd550 100644 --- a/src/util.h +++ b/src/util.h @@ -241,7 +241,8 @@ bool SoftSetArg(const std::string& strArg, const std::string& strValue); */ bool SoftSetBoolArg(const std::string& strArg, bool fValue); -// Forces a arg setting, used only in testing +// Forces an arg setting. Called by SoftSetArg() if the arg hasn't already +// been set. Also called directly in testing. void ForceSetArg(const std::string& strArg, const std::string& strValue); }; diff --git a/src/validation.cpp b/src/validation.cpp index 73466b9df7..de65839eef 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -268,15 +268,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -309,16 +309,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool return EvaluateSequenceLocks(index, lockPair); } - void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { int expired = pool.Expire(GetTime() - age); if (expired != 0) { LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } - std::vector<uint256> vNoSpendsRemaining; + std::vector<COutPoint> vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining) pcoinsTip->Uncache(removed); } @@ -343,9 +342,59 @@ static bool IsCurrentForFeeEstimation() return true; } +/* Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + +void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool fAddToMempool) +{ + AssertLockHeld(cs_main); + std::vector<uint256> vHashUpdate; + // disconnectpool's insertion_order index sorts the entries from + // oldest to newest, but the oldest entry will be the last tx from the + // latest mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions + // back to the mempool starting with the earliest transaction that had + // been previously seen in a block. + auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); + while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { + // ignore validation errors in resurrected transactions + CValidationState stateDummy; + if (!fAddToMempool || (*it)->IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, *it, false, NULL, NULL, true)) { + // If the transaction doesn't make it in to the mempool, remove any + // transactions that depend on it (which would now be orphans). + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (mempool.exists((*it)->GetHash())) { + vHashUpdate.push_back((*it)->GetHash()); + } + ++it; + } + disconnectpool.queuedTx.clear(); + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in + // the disconnectpool that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + + // We also need to remove any now-immature transactions + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + // Re-limit mempool size, in case we added any transactions + LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); +} + bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -438,30 +487,30 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C view.SetBackend(viewMemPool); // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + for (size_t out = 0; out < tx.vout.size(); out++) { + COutPoint outpoint(hash, out); + bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); + if (view.HaveCoin(outpoint)) { + if (!had_coin_in_cache) { + coins_to_uncache.push_back(outpoint); + } + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } } // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) + if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); + } + if (!view.HaveCoin(txin.prevout)) { + if (pfMissingInputs) { *pfMissingInputs = true; + } return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - // Bring the best block into scope view.GetBestBlock(); @@ -499,8 +548,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { + const Coin &coin = view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } @@ -764,10 +813,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector<COutPoint> coins_to_uncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -819,15 +868,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus } if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1075,24 +1117,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; - } + txundo.vprevout.emplace_back(); + inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1136,11 +1166,19 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + const CScript& scriptPubKey = coin.out.scriptPubKey; + const CAmount amount = coin.out.nValue; // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); + CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); @@ -1152,7 +1190,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, + CScriptCheck check2(scriptPubKey, amount, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); @@ -1211,8 +1249,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << hashBlock; + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1220,10 +1260,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; @@ -1249,46 +1286,43 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // anon namespace +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + view.AddCoin(out, std::move(undo), undo.fCoinBase); - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -enum DisconnectResult -{ - DISCONNECT_OK, // All good. - DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. - DISCONNECT_FAILED // Something else went wrong. -}; - /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */ static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) @@ -1320,36 +1354,31 @@ static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + view.SpendCoin(out, &coin); + if (tx.vout[o] != coin.out) { + fClean = false; // transaction output mismatch + } + } } // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; + CTxUndo &txundo = blockUndo.vtxundo[i-1]; if (txundo.vprevout.size() != tx.vin.size()) { error("DisconnectBlock(): transaction and undo data inconsistent"); return DISCONNECT_FAILED; } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } @@ -1530,10 +1559,12 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (fEnforceBIP30) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + REJECT_INVALID, "bad-txns-BIP30"); + } + } } } @@ -1600,7 +1631,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { @@ -1738,9 +1769,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), - std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -1781,12 +1811,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) @@ -1815,6 +1845,16 @@ void PruneAndFlush() { FlushStateToDisk(state, FLUSH_STATE_NONE); } +static void DoWarning(const std::string& strWarning) +{ + static bool fWarned = false; + SetMiscWarning(strWarning); + if (!fWarned) { + AlertNotify(strWarning); + fWarned = true; + } +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); @@ -1824,7 +1864,6 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { cvBlockChange.notify_all(); - static bool fWarned = false; std::vector<std::string> warningMessages; if (!IsInitialBlockDownload()) { @@ -1834,15 +1873,11 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { + const std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); if (state == THRESHOLD_ACTIVE) { - std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } else { - warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); + warningMessages.push_back(strWarning); } } } @@ -1855,19 +1890,15 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { pindex = pindex->pprev; } if (nUpgraded > 0) - warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); + warningMessages.push_back(strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); if (nUpgraded > 100/2) { std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), @@ -1878,8 +1909,17 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } -/** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) +/** Disconnect chainActive's tip. + * After calling, the mempool will be in an inconsistent state, with + * transactions from disconnected blocks being added to disconnectpool. You + * should make the mempool consistent again by calling UpdateMempoolForReorg. + * with cs_main held. + * + * If disconnectpool is NULL, then no disconnected transactions are added to + * disconnectpool (note that the caller is responsible for mempool consistency + * in any case). + */ +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -1902,25 +1942,17 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - if (!fBare) { - // Resurrect mempool transactions from the disconnected block. - std::vector<uint256> vHashUpdate; - for (const auto& it : block.vtx) { - const CTransaction& tx = *it; - // ignore validation errors in resurrected transactions - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { - mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } + if (disconnectpool) { + // Save transactions to re-add to mempool at end of reorg + for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { + disconnectpool->addTransaction(*it); + } + while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { + // Drop the earliest entry, and remove its children from the mempool. + auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + disconnectpool->removeEntry(it); } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); } // Update chainActive and related variables. @@ -2008,7 +2040,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2050,6 +2082,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); @@ -2143,9 +2176,14 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; + DisconnectedBlockTransactions disconnectpool; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams)) + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // This is likely a fatal error, but keep the mempool consistent, + // just in case. Only remove from the mempool in this case. + UpdateMempoolForReorg(disconnectpool, false); return false; + } fBlocksDisconnected = true; } @@ -2168,7 +2206,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) @@ -2179,6 +2217,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c break; } else { // A system error occurred (disk space, database error, ...). + // Make the mempool consistent with the current tip, just in case + // any observers try to use it before shutdown. + UpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2193,8 +2234,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c } if (fBlocksDisconnected) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // If any blocks were disconnected, disconnectpool may be non empty. Add + // any disconnected transactions back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); } mempool.check(pcoinsTip); @@ -2343,6 +2385,7 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); + DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { CBlockIndex *pindexWalk = chainActive.Tip(); pindexWalk->nStatus |= BLOCK_FAILED_CHILD; @@ -2350,13 +2393,17 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams)) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // It's probably hopeless to try to make the mempool consistent + // here if DisconnectTip failed, but we can try. + UpdateMempoolForReorg(disconnectpool, false); return false; } } - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // DisconnectTip will add transactions to disconnectpool; try to add these + // back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. @@ -2369,7 +2416,6 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C } InvalidChainFound(pindex); - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } @@ -2817,8 +2863,8 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam if (!fHaveWitness) { - for (size_t i = 0; i < block.vtx.size(); i++) { - if (block.vtx[i]->HasWitness()) { + for (const auto& tx : block.vtx) { + if (tx->HasWitness()) { return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); } } @@ -3483,7 +3529,7 @@ bool RewindBlockIndex(const CChainParams& params) // of the blockchain). break; } - if (!DisconnectTip(state, params, true)) { + if (!DisconnectTip(state, params, NULL)) { return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); } // Occasionally flush state to disk. @@ -3935,6 +3981,12 @@ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::D return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); } +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + LOCK(cs_main); + return VersionBitsStatistics(chainActive.Tip(), params, pos); +} + int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); diff --git a/src/validation.h b/src/validation.h index 743a2973f8..096fd0a9ee 100644 --- a/src/validation.h +++ b/src/validation.h @@ -70,6 +70,8 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -105,7 +107,7 @@ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; /** Average delay between local address broadcasts in seconds. */ -static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; +static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; /** Average delay between peer address broadcasts in seconds. */ static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; /** Average delay between trickled inventory transmissions in seconds. @@ -339,6 +341,9 @@ std::string FormatStateMessage(const CValidationState &state); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); +/** Get the numerical statistics for the BIP9 state for a given deployment at the current tip. */ +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos); + /** Get the block height at which the BIP9 deployment switched into the state for the block building on the current tip. */ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos); @@ -401,8 +406,8 @@ private: public: CScriptCheck(): amount(0), ptxTo(0), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR) {} - CScriptCheck(const CCoins& txFromIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : - scriptPubKey(txFromIn.vout[txToIn.vin[nInIn].prevout.n].scriptPubKey), amount(txFromIn.vout[txToIn.vin[nInIn].prevout.n].nValue), + CScriptCheck(const CScript& scriptPubKeyIn, const CAmount amountIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : + scriptPubKey(scriptPubKeyIn), amount(amountIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn) { } bool operator()(); diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 8a7cce7485..4bb352f23c 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -3,10 +3,9 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "versionbits.h" - #include "consensus/params.h" -const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { +const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { { /*.name =*/ "testdummy", /*.gbt_force =*/ true, @@ -105,6 +104,36 @@ ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* return state; } +// return the numerical statistics of blocks signalling the specified BIP9 condition in this current period +BIP9Stats AbstractThresholdConditionChecker::GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const +{ + BIP9Stats stats; + + stats.period = Period(params); + stats.threshold = Threshold(params); + + if (pindex == NULL) + return stats; + + // Find beginning of period + const CBlockIndex* pindexEndOfPrevPeriod = pindex->GetAncestor(pindex->nHeight - ((pindex->nHeight + 1) % stats.period)); + stats.elapsed = pindex->nHeight - pindexEndOfPrevPeriod->nHeight; + + // Count from current block to beginning of period + int count = 0; + const CBlockIndex* currentIndex = pindex; + while (pindexEndOfPrevPeriod->nHeight != currentIndex->nHeight){ + if (Condition(currentIndex, params)) + count++; + currentIndex = currentIndex->pprev; + } + + stats.count = count; + stats.possible = (stats.period - stats.threshold ) >= (stats.elapsed - count); + + return stats; +} + int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { const ThresholdState initialState = GetStateFor(pindexPrev, params, cache); @@ -167,6 +196,11 @@ ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus:: return VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, cache.caches[pos]); } +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + return VersionBitsConditionChecker(pos).GetStateStatisticsFor(pindexPrev, params); +} + int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache) { return VersionBitsConditionChecker(pos).GetStateSinceHeightFor(pindexPrev, params, cache.caches[pos]); diff --git a/src/versionbits.h b/src/versionbits.h index 7a929266aa..f4dfb71515 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -30,14 +30,22 @@ enum ThresholdState { // will either be NULL or a block with (height + 1) % Period() == 0. typedef std::map<const CBlockIndex*, ThresholdState> ThresholdConditionCache; -struct BIP9DeploymentInfo { +struct VBDeploymentInfo { /** Deployment name */ const char *name; /** Whether GBT clients can safely ignore this rule in simplified usage */ bool gbt_force; }; -extern const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[]; +struct BIP9Stats { + int period; + int threshold; + int elapsed; + int count; + bool possible; +}; + +extern const struct VBDeploymentInfo VersionBitsDeploymentInfo[]; /** * Abstract class that implements BIP9-style threshold logic, and caches results. @@ -51,6 +59,7 @@ protected: virtual int Threshold(const Consensus::Params& params) const =0; public: + BIP9Stats GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const; // Note that the functions below take a pindexPrev as input: they compute information for block B based on its parent. ThresholdState GetStateFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; int GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; @@ -64,6 +73,7 @@ struct VersionBitsCache }; ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); uint32_t VersionBitsMask(const Consensus::Params& params, Consensus::DeploymentPos pos); diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 2aa26fb00a..cb4719ae90 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -18,8 +18,6 @@ public: bool fAllowOtherInputs; //! Includes watch only addresses which match the ISMINE_WATCH_SOLVABLE criteria bool fAllowWatchOnly; - //! Minimum absolute fee (not per kilobyte) - CAmount nMinimumTotalFee; //! Override estimated feerate bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true @@ -40,7 +38,6 @@ public: fAllowOtherInputs = false; fAllowWatchOnly = false; setSelected.clear(); - nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; nConfirmTarget = 0; diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index c10a9eccd9..46ef87b7b1 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -66,7 +66,7 @@ bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx return true; } -CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable) +CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable) : txid(std::move(txidIn)), nOldFee(0), @@ -165,15 +165,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf nNewFee = totalFee; nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); } else { - // if user specified a confirm target then don't consider any global payTxFee - if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, true); - } - // otherwise use the regular wallet logic to select payTxFee or default confirm target - else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator); - } - + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, ignoreGlobalPayTxFee); nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate @@ -255,7 +247,7 @@ bool CFeeBumper::commit(CWallet *pWallet) } CWalletTx& oldWtx = pWallet->mapWallet[txid]; - // make sure the transaction still has no descendants and hasen't been mined in the meantime + // make sure the transaction still has no descendants and hasn't been mined in the meantime if (!preconditionChecks(pWallet, oldWtx)) { return false; } diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index f40d05da28..fc32316704 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -24,7 +24,7 @@ enum class BumpFeeResult class CFeeBumper { public: - CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable); + CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable); BumpFeeResult getResult() const { return currentResult; } const std::vector<std::string>& getErrors() const { return vErrors; } CAmount getOldFee() const { return nOldFee; } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 82708dab26..457e937bb8 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -357,7 +357,7 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "removeprunedfunds \"txid\"\n" - "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will effect wallet balances.\n" + "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will affect wallet balances.\n" "\nArguments:\n" "1. \"txid\" (string, required) The hex-encoded id of the transaction you are deleting\n" "\nExamples:\n" @@ -536,14 +536,11 @@ UniValue importwallet(const JSONRPCRequest& request) } file.close(); pwallet->ShowProgress("", 100); // hide progress dialog in GUI - - CBlockIndex *pindex = chainActive.Tip(); - while (pindex && pindex->pprev && pindex->GetBlockTime() > nTimeBegin - TIMESTAMP_WINDOW) - pindex = pindex->pprev; - pwallet->UpdateTimeFirstKey(nTimeBegin); - LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); + CBlockIndex *pindex = chainActive.FindEarliestAtLeast(nTimeBegin - TIMESTAMP_WINDOW); + + LogPrintf("Rescanning last %i blocks\n", pindex ? chainActive.Height() - pindex->nHeight + 1 : 0); pwallet->ScanForWalletTransactions(pindex); pwallet->MarkDirty(); @@ -606,7 +603,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) "dumpwallet \"filename\"\n" "\nDumps all wallet keys in a human-readable format.\n" "\nArguments:\n" - "1. \"filename\" (string, required) The filename\n" + "1. \"filename\" (string, required) The filename with path (either absolute or relative to bitcoind)\n" + "\nResult:\n" + "{ (json object)\n" + " \"filename\" : { (string) The filename with full absolute path\n" + "}\n" "\nExamples:\n" + HelpExampleCli("dumpwallet", "\"test\"") + HelpExampleRpc("dumpwallet", "\"test\"") @@ -617,7 +618,9 @@ UniValue dumpwallet(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); std::ofstream file; - file.open(request.params[0].get_str().c_str()); + boost::filesystem::path filepath = request.params[0].get_str(); + filepath = boost::filesystem::absolute(filepath); + file.open(filepath.string().c_str()); if (!file.is_open()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); @@ -682,7 +685,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) file << "\n"; file << "# End of dump\n"; file.close(); - return NullUniValue; + + UniValue reply(UniValue::VOBJ); + reply.push_back(Pair("filename", filepath.string())); + + return reply; } @@ -1121,13 +1128,13 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) if (fRescan && fRunScan && requests.size()) { CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindEarliestAtLeast(std::max<int64_t>(nLowestTimestamp - TIMESTAMP_WINDOW, 0)) : chainActive.Genesis(); - CBlockIndex* scannedRange = nullptr; + CBlockIndex* scanFailed = nullptr; if (pindex) { - scannedRange = pwallet->ScanForWalletTransactions(pindex, true); + scanFailed = pwallet->ScanForWalletTransactions(pindex, true); pwallet->ReacceptWalletTransactions(); } - if (!scannedRange || scannedRange->nHeight > pindex->nHeight) { + if (scanFailed) { std::vector<UniValue> results = response.getValues(); response.clear(); response.setArray(); @@ -1137,12 +1144,23 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) // range, or if the import result already has an error set, let // the result stand unmodified. Otherwise replace the result // with an error message. - if (GetImportTimestamp(request, now) - TIMESTAMP_WINDOW >= scannedRange->GetBlockTimeMax() || results.at(i).exists("error")) { + if (GetImportTimestamp(request, now) - TIMESTAMP_WINDOW > scanFailed->GetBlockTimeMax() || results.at(i).exists("error")) { response.push_back(results.at(i)); } else { UniValue result = UniValue(UniValue::VOBJ); result.pushKV("success", UniValue(false)); - result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, strprintf("Failed to rescan before time %d, transactions may be missing.", scannedRange->GetBlockTimeMax()))); + result.pushKV( + "error", + JSONRPCError( + RPC_MISC_ERROR, + strprintf("Rescan failed for key with creation timestamp %d. There was an error reading a " + "block from time %d, which is after or within %d seconds of key creation, and " + "could contain transactions pertaining to the key. As a result, transactions " + "and coins using this key may not appear in the wallet. This error could be " + "caused by pruning or data corruption (see bitcoind log for details) and could " + "be dealt with by downloading and rescanning the relevant blocks (see -reindex " + "and -rescan options).", + GetImportTimestamp(request, now), scanFailed->GetBlockTimeMax(), TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } ++i; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index ae4f4f37cb..3b6dfd42ca 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2834,7 +2834,7 @@ UniValue bumpfee(const JSONRPCRequest& request) hash.SetHex(request.params[0].get_str()); // optional parameters - bool specifiedConfirmTarget = false; + bool ignoreGlobalPayTxFee = false; int newConfirmTarget = nTxConfirmTarget; CAmount totalFee = 0; bool replaceable = true; @@ -2851,7 +2851,10 @@ UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("confTarget") && options.exists("totalFee")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and totalFee options should not both be set. Please provide either a confirmation target for fee estimation or an explicit total fee for the transaction."); } else if (options.exists("confTarget")) { - specifiedConfirmTarget = true; + // If the user has explicitly set a confTarget in this rpc call, + // then override the default logic that uses the global payTxFee + // instead of the confirmation target. + ignoreGlobalPayTxFee = true; newConfirmTarget = options["confTarget"].get_int(); if (newConfirmTarget <= 0) { // upper-bound will be checked by estimatefee/smartfee throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid confTarget (cannot be <= 0)"); @@ -2871,7 +2874,7 @@ UniValue bumpfee(const JSONRPCRequest& request) LOCK2(cs_main, pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - CFeeBumper feeBump(pwallet, hash, newConfirmTarget, specifiedConfirmTarget, totalFee, replaceable); + CFeeBumper feeBump(pwallet, hash, newConfirmTarget, ignoreGlobalPayTxFee, totalFee, replaceable); BumpFeeResult res = feeBump.getResult(); if (res != BumpFeeResult::OK) { @@ -2909,7 +2912,7 @@ UniValue bumpfee(const JSONRPCRequest& request) UniValue errors(UniValue::VARR); for (const std::string& err: feeBump.getErrors()) errors.push_back(err); - result.push_back(errors); + result.push_back(Pair("errors", errors)); return result; } diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 8eeba72a06..b30748d66b 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -9,6 +9,7 @@ #include <utility> #include <vector> +#include "consensus/validation.h" #include "rpc/server.h" #include "test/test_bitcoin.h" #include "validation.h" @@ -19,6 +20,8 @@ #include <univalue.h> extern UniValue importmulti(const JSONRPCRequest& request); +extern UniValue dumpwallet(const JSONRPCRequest& request); +extern UniValue importwallet(const JSONRPCRequest& request); // how many times to run all the tests to have a chance to catch errors that only show up with particular random shuffles #define RUN_TESTS 100 @@ -364,6 +367,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) LOCK(cs_main); // Cap last block file size, and mine new block in a new block file. + CBlockIndex* const nullBlock = nullptr; CBlockIndex* oldTip = chainActive.Tip(); GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE; CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); @@ -375,7 +379,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CWallet wallet; LOCK(wallet.cs_wallet); wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); } @@ -389,7 +393,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CWallet wallet; LOCK(wallet.cs_wallet); wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); } @@ -413,7 +417,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CKey futureKey; futureKey.MakeNewKey(true); key.pushKV("scriptPubKey", HexStr(GetScriptForRawPubKey(futureKey.GetPubKey()))); - key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW); + key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); keys.push_back(key); JSONRPCRequest request; @@ -421,20 +425,77 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) request.params.push_back(keys); UniValue response = importmulti(request); - BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); + BOOST_CHECK_EQUAL(response.write(), + strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Rescan failed for key with creation " + "timestamp %d. There was an error reading a block from time %d, which is after or within %d " + "seconds of key creation, and could contain transactions pertaining to the key. As a result, " + "transactions and coins using this key may not appear in the wallet. This error could be caused " + "by pruning or data corruption (see bitcoind log for details) and could be dealt with by " + "downloading and rescanning the relevant blocks (see -reindex and -rescan " + "options).\"}},{\"success\":true}]", + 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); ::pwalletMain = backup; } +} + +// Verify importwallet RPC starts rescan at earliest block with timestamp +// greater or equal than key birthday. Previously there was a bug where +// importwallet RPC would start the scan at the latest block with timestamp less +// than or equal to key birthday. +BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) +{ + CWallet *pwalletMainBackup = ::pwalletMain; + LOCK(cs_main); - // Verify ScanForWalletTransactions does not return null when the scan is - // elided due to the nTimeFirstKey optimization. + // Create two blocks with same timestamp to verify that importwallet rescan + // will pick up both blocks, not just the first. + const int64_t BLOCK_TIME = chainActive.Tip()->GetBlockTimeMax() + 5; + SetMockTime(BLOCK_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Set key birthday to block time increased by the timestamp window, so + // rescan will start at the block time. + const int64_t KEY_TIME = BLOCK_TIME + TIMESTAMP_WINDOW; + SetMockTime(KEY_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Import key into wallet and call dumpwallet to create backup file. { CWallet wallet; - { - LOCK(wallet.cs_wallet); - wallet.UpdateTimeFirstKey(newTip->GetBlockTime() + 7200 + 1); + LOCK(wallet.cs_wallet); + wallet.mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::dumpwallet(request); + } + + // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME + // were scanned, and no prior blocks were scanned. + { + CWallet wallet; + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::importwallet(request); + + BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); + BOOST_CHECK_EQUAL(coinbaseTxns.size(), 103); + for (size_t i = 0; i < coinbaseTxns.size(); ++i) { + bool found = wallet.GetWalletTx(coinbaseTxns[i].GetHash()); + bool expected = i >= 100; + BOOST_CHECK_EQUAL(found, expected); } - BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(newTip)); } + + SetMockTime(0); + ::pwalletMain = pwalletMainBackup; } // Check that GetImmatureCredit() returns a newly calculated value instead of @@ -515,4 +576,104 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) SetMockTime(0); } +BOOST_AUTO_TEST_CASE(LoadReceiveRequests) +{ + CTxDestination dest = CKeyID(); + pwalletMain->AddDestData(dest, "misc", "val_misc"); + pwalletMain->AddDestData(dest, "rr0", "val_rr0"); + pwalletMain->AddDestData(dest, "rr1", "val_rr1"); + + auto values = pwalletMain->GetDestValues("rr"); + BOOST_CHECK_EQUAL(values.size(), 2); + BOOST_CHECK_EQUAL(values[0], "val_rr0"); + BOOST_CHECK_EQUAL(values[1], "val_rr1"); +} + +class ListCoinsTestingSetup : public TestChain100Setup +{ +public: + ListCoinsTestingSetup() + { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + ::bitdb.MakeMock(); + wallet.reset(new CWallet(std::unique_ptr<CWalletDBWrapper>(new CWalletDBWrapper(&bitdb, "wallet_test.dat")))); + bool firstRun; + wallet->LoadWallet(firstRun); + LOCK(wallet->cs_wallet); + wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + wallet->ScanForWalletTransactions(chainActive.Genesis()); + } + + ~ListCoinsTestingSetup() + { + wallet.reset(); + ::bitdb.Flush(true); + ::bitdb.Reset(); + } + + CWalletTx& AddTx(CRecipient recipient) + { + CWalletTx wtx; + CReserveKey reservekey(wallet.get()); + CAmount fee; + int changePos = -1; + std::string error; + BOOST_CHECK(wallet->CreateTransaction({recipient}, wtx, reservekey, fee, changePos, error)); + CValidationState state; + BOOST_CHECK(wallet->CommitTransaction(wtx, reservekey, nullptr, state)); + auto it = wallet->mapWallet.find(wtx.GetHash()); + BOOST_CHECK(it != wallet->mapWallet.end()); + CreateAndProcessBlock({CMutableTransaction(*it->second.tx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + it->second.SetMerkleBranch(chainActive.Tip(), 1); + return it->second; + } + + std::unique_ptr<CWallet> wallet; +}; + +BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) +{ + std::string coinbaseAddress = coinbaseKey.GetPubKey().GetID().ToString(); + LOCK(wallet->cs_wallet); + + // Confirm ListCoins initially returns 1 coin grouped under coinbaseKey + // address. + auto list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 1); + + // Check initial balance from one mature coinbase transaction. + BOOST_CHECK_EQUAL(50 * COIN, wallet->GetAvailableBalance()); + + // Add a transaction creating a change address, and confirm ListCoins still + // returns the coin associated with the change address underneath the + // coinbaseKey pubkey, even though the change address has a different + // pubkey. + AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); + + // Lock both coins. Confirm number of available coins drops to 0. + std::vector<COutput> available; + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 2); + for (const auto& group : list) { + for (const auto& coin : group.second) { + wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); + } + } + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 0); + + // Confirm ListCoins still returns same result as before, despite coins + // being locked. + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 9eba8ad9fe..3e000d2a9d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -982,6 +982,13 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI return false; } +bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const +{ + LOCK2(cs_main, cs_wallet); + const CWalletTx* wtx = GetWalletTx(hashTx); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() <= 0 && !wtx->InMempool(); +} + bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); @@ -1456,10 +1463,9 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, * from or to us. If fUpdate is true, found transactions that already * exist in the wallet will be updated. * - * Returns pointer to the first block in the last contiguous range that was - * successfully scanned or elided (elided if pIndexStart points at a block - * before CWallet::nTimeFirstKey). Returns null if there is no such range, or - * the range doesn't include chainActive.Tip(). + * Returns null if scan was successful. Otherwise, if a complete rescan was not + * possible (due to pruning or corruption), returns pointer to the most recent + * block that could not be scanned. */ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) { @@ -1467,7 +1473,7 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f const CChainParams& chainParams = Params(); CBlockIndex* pindex = pindexStart; - CBlockIndex* ret = pindexStart; + CBlockIndex* ret = nullptr; { LOCK2(cs_main, cs_wallet); fAbortRescan = false; @@ -1495,11 +1501,8 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate); } - if (!ret) { - ret = pindex; - } } else { - ret = nullptr; + ret = pindex; } pindex = chainActive.Next(pindex); } @@ -1781,8 +1784,8 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { CMutableTransaction tx1 = *this->tx; CMutableTransaction tx2 = *_tx.tx; - for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); - for (unsigned int i = 0; i < tx2.vin.size(); i++) tx2.vin[i].scriptSig = CScript(); + for (auto& txin : tx1.vin) txin.scriptSig = CScript(); + for (auto& txin : tx2.vin) txin.scriptSig = CScript(); return CTransaction(tx1) == CTransaction(tx2); } @@ -1977,6 +1980,21 @@ CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth, cons return balance; } +CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + std::vector<COutput> vCoins; + AvailableCoins(vCoins, true, coinControl); + for (const COutput& out : vCoins) { + if (out.fSpendable) { + balance += out.tx->tx->vout[out.i].nValue; + } + } + return balance; +} + void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const { vCoins.clear(); @@ -2088,6 +2106,69 @@ void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const } } +std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const +{ + // TODO: Add AssertLockHeld(cs_wallet) here. + // + // Because the return value from this function contains pointers to + // CWalletTx objects, callers to this function really should acquire the + // cs_wallet lock before calling it. However, the current caller doesn't + // acquire this lock yet. There was an attempt to add the missing lock in + // https://github.com/bitcoin/bitcoin/pull/10340, but that change has been + // postponed until after https://github.com/bitcoin/bitcoin/pull/10244 to + // avoid adding some extra complexity to the Qt code. + + std::map<CTxDestination, std::vector<COutput>> result; + + std::vector<COutput> availableCoins; + AvailableCoins(availableCoins); + + LOCK2(cs_main, cs_wallet); + for (auto& coin : availableCoins) { + CTxDestination address; + if (coin.fSpendable && + ExtractDestination(FindNonChangeParentOutput(*coin.tx->tx, coin.i).scriptPubKey, address)) { + result[address].emplace_back(std::move(coin)); + } + } + + std::vector<COutPoint> lockedCoins; + ListLockedCoins(lockedCoins); + for (const auto& output : lockedCoins) { + auto it = mapWallet.find(output.hash); + if (it != mapWallet.end()) { + int depth = it->second.GetDepthInMainChain(); + if (depth >= 0 && output.n < it->second.tx->vout.size() && + IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { + CTxDestination address; + if (ExtractDestination(FindNonChangeParentOutput(*it->second.tx, output.n).scriptPubKey, address)) { + result[address].emplace_back( + &it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + } + } + } + } + + return result; +} + +const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const +{ + const CTransaction* ptx = &tx; + int n = output; + while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) { + const COutPoint& prevout = ptx->vin[0].prevout; + auto it = mapWallet.find(prevout.hash); + if (it == mapWallet.end() || it->second.tx->vout.size() <= prevout.n || + !IsMine(it->second.tx->vout[prevout.n])) { + break; + } + ptx = it->second.tx.get(); + n = prevout.n; + } + return ptx->vout[n]; +} + static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { @@ -2183,10 +2264,10 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin if (nTotalLower == nTargetValue) { - for (unsigned int i = 0; i < vValue.size(); ++i) + for (const auto& input : vValue) { - setCoinsRet.insert(vValue[i]); - nValueRet += vValue[i].txout.nValue; + setCoinsRet.insert(input); + nValueRet += input.txout.nValue; } return true; } @@ -2316,7 +2397,7 @@ bool CWallet::SignTransaction(CMutableTransaction &tx) // sign the new tx CTransaction txNewConst(tx); int nIn = 0; - for (auto& input : tx.vin) { + for (const auto& input : tx.vin) { std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(input.prevout.hash); if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { return false; @@ -2635,9 +2716,6 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT currentConfirmationTarget = coinControl->nConfirmTarget; CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); - if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { - nFeeNeeded = coinControl->nMinimumTotalFee; - } if (coinControl && coinControl->fOverrideFeeRate) nFeeNeeded = coinControl->nFeeRate.GetFee(nBytes); @@ -2808,12 +2886,12 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreUserSetFee) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreGlobalPayTxFee) { // payTxFee is the user-set global for desired feerate CAmount nFeeNeeded = payTxFee.GetFee(nTxBytes); // User didn't set: use -txconfirmtarget to estimate... - if (nFeeNeeded == 0 || ignoreUserSetFee) { + if (nFeeNeeded == 0 || ignoreGlobalPayTxFee) { int estimateFoundTarget = nConfirmTarget; nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee @@ -3179,9 +3257,9 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() { LOCK(cs_wallet); - BOOST_FOREACH(PAIRTYPE(uint256, CWalletTx) walletEntry, mapWallet) + for (const auto& walletEntry : mapWallet) { - CWalletTx *pcoin = &walletEntry.second; + const CWalletTx *pcoin = &walletEntry.second; if (!pcoin->IsTrusted()) continue; @@ -3219,9 +3297,9 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() std::set< std::set<CTxDestination> > groupings; std::set<CTxDestination> grouping; - BOOST_FOREACH(PAIRTYPE(uint256, CWalletTx) walletEntry, mapWallet) + for (const auto& walletEntry : mapWallet) { - CWalletTx *pcoin = &walletEntry.second; + const CWalletTx *pcoin = &walletEntry.second; if (pcoin->tx->vin.size() > 0) { @@ -3258,11 +3336,11 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) - if (IsMine(pcoin->tx->vout[i])) + for (const auto& txout : pcoin->tx->vout) + if (IsMine(txout)) { CTxDestination address; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, address)) + if(!ExtractDestination(txout.scriptPubKey, address)) continue; grouping.insert(address); groupings.insert(grouping); @@ -3410,7 +3488,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const return (setLockedCoins.count(outpt) > 0); } -void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) +void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { AssertLockHeld(cs_wallet); // setLockedCoins for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); @@ -3611,6 +3689,20 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st return false; } +std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const +{ + LOCK(cs_wallet); + std::vector<std::string> values; + for (const auto& address : mapAddressBook) { + for (const auto& data : address.second.destdata) { + if (!data.first.compare(0, prefix.size(), prefix)) { + values.emplace_back(data.second); + } + } + } + return values; +} + std::string CWallet::GetWalletHelpString(bool showDebug) { std::string strUsage = HelpMessageGroup(_("Wallet options:")); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 69f51b3f64..a3974bf00b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -821,6 +821,16 @@ public: void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; /** + * Return list of available coins and locked coins grouped by non-change output address. + */ + std::map<CTxDestination, std::vector<COutput>> ListCoins() const; + + /** + * Find non-change parent output. + */ + const CTxOut& FindNonChangeParentOutput(const CTransaction& tx, int output) const; + + /** * Shuffle and select coins until nTargetValue is reached while avoiding * small change; This method is stochastic for some inputs and upon * completion the coin set and corresponding actual target value is @@ -834,7 +844,7 @@ public: void LockCoin(const COutPoint& output); void UnlockCoin(const COutPoint& output); void UnlockAllCoins(); - void ListLockedCoins(std::vector<COutPoint>& vOutpts); + void ListLockedCoins(std::vector<COutPoint>& vOutpts) const; /* * Rescan abort properties @@ -873,6 +883,8 @@ public: bool LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value); //! Look up a destination data tuple in the store, return true if found false otherwise bool GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const; + //! Get all destination values matching a prefix. + std::vector<std::string> GetDestValues(const std::string& prefix) const; //! Adds a watch-only address to the store, and saves it to disk. bool AddWatchOnly(const CScript& dest, int64_t nCreateTime); @@ -917,6 +929,7 @@ public: CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; CAmount GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const; + CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; /** * Insert additional inputs into the transaction by @@ -946,7 +959,7 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreUserSetFee = false); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreGlobalPayTxFee = false); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -1066,6 +1079,9 @@ public: /** Set whether this wallet broadcasts transactions. */ void SetBroadcastTransactions(bool broadcast) { fBroadcastTransactions = broadcast; } + /** Return whether transaction can be abandoned */ + bool TransactionCanBeAbandoned(const uint256& hashTx) const; + /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ bool AbandonTransaction(const uint256& hashTx); diff --git a/test/functional/abandonconflict.py b/test/functional/abandonconflict.py index 9748757641..c87c02492d 100755 --- a/test/functional/abandonconflict.py +++ b/test/functional/abandonconflict.py @@ -73,8 +73,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Restart the node with a higher min relay fee so the parent tx is no longer in mempool # TODO: redo with eviction - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) # Verify txs no longer in either node's mempool assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -100,8 +100,8 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.00001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.00001"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(self.nodes[0].getbalance(), balance) @@ -120,8 +120,8 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Remove using high relay fee again - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) diff --git a/test/functional/assumevalid.py b/test/functional/assumevalid.py index 8e301c4379..9d17faac51 100755 --- a/test/functional/assumevalid.py +++ b/test/functional/assumevalid.py @@ -45,7 +45,7 @@ from test_framework.mininode import (CBlockHeader, msg_headers) from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_node, p2p_port, assert_equal) +from test_framework.util import (p2p_port, assert_equal) class BaseNode(NodeConnCB): def send_header_for_blocks(self, new_blocks): @@ -63,7 +63,7 @@ class AssumeValidTest(BitcoinTestFramework): # Start node0. We don't start the other nodes yet since # we need to pre-mine a block with an invalid transaction # signature so we can pass in the block hash as assumevalid. - self.nodes = [start_node(0, self.options.tmpdir)] + self.nodes = [self.start_node(0, self.options.tmpdir)] def send_blocks_until_disconnected(self, node): """Keep sending blocks to the node until we're disconnected.""" @@ -162,14 +162,14 @@ class AssumeValidTest(BitcoinTestFramework): height += 1 # Start node1 and node2 with assumevalid so they accept a block with a bad signature. - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-assumevalid=" + hex(block102.sha256)])) node1 = BaseNode() # connects to node1 connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], node1)) node1.add_connection(connections[1]) node1.wait_for_verack() - self.nodes.append(start_node(2, self.options.tmpdir, + self.nodes.append(self.start_node(2, self.options.tmpdir, ["-assumevalid=" + hex(block102.sha256)])) node2 = BaseNode() # connects to node2 connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) diff --git a/test/functional/bip9-softforks.py b/test/functional/bip9-softforks.py index 1b2dff63d2..b90b0ca628 100755 --- a/test/functional/bip9-softforks.py +++ b/test/functional/bip9-softforks.py @@ -99,12 +99,43 @@ class BIP9SoftForksTest(ComparisonTestFramework): assert_equal(self.get_bip9_status(bipName)['status'], 'started') assert_equal(self.get_bip9_status(bipName)['since'], 144) + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) assert_equal(tmpl['vbavailable'][bipName], bitno) assert_equal(tmpl['vbrequired'], 0) assert(tmpl['version'] & activated_version) + # Test 1-A + # check stats after max number of "signalling not" blocks such that LOCKED_IN still possible this period + test_blocks = self.generate_blocks(36, 4, test_blocks) # 0x00000004 (signalling not) + test_blocks = self.generate_blocks(10, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 46) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 10) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + + # Test 1-B + # check stats after one additional "signalling not" block -- LOCKED_IN no longer possible this period + test_blocks = self.generate_blocks(1, 4, test_blocks) # 0x00000004 (signalling not) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 47) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 10) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], False) + + # Test 1-C + # finish period with "ready" blocks, but soft fork will still fail to advance to LOCKED_IN + test_blocks = self.generate_blocks(97, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + assert_equal(self.get_bip9_status(bipName)['status'], 'started') + # Test 2 # Fail to achieve LOCKED_IN 100 out of 144 signal bit 1 # using a variety of bits to simulate multiple parallel softforks @@ -116,6 +147,8 @@ class BIP9SoftForksTest(ComparisonTestFramework): assert_equal(self.get_bip9_status(bipName)['status'], 'started') assert_equal(self.get_bip9_status(bipName)['since'], 144) + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) assert_equal(tmpl['vbavailable'][bipName], bitno) @@ -125,14 +158,24 @@ class BIP9SoftForksTest(ComparisonTestFramework): # Test 3 # 108 out of 144 signal bit 1 to achieve LOCKED_IN # using a variety of bits to simulate multiple parallel softforks - test_blocks = self.generate_blocks(58, activated_version) # 0x20000001 (signalling ready) + test_blocks = self.generate_blocks(57, activated_version) # 0x20000001 (signalling ready) test_blocks = self.generate_blocks(26, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, activated_version, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(10, 4, test_blocks) # 0x20010000 (signalling not) yield TestInstance(test_blocks, sync_every_block=False) + # check counting stats and "possible" flag before last block of this period achieves LOCKED_IN... + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 143) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 107) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + assert_equal(self.get_bip9_status(bipName)['status'], 'started') + + # ...continue with Test 3 + test_blocks = self.generate_blocks(1, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') - assert_equal(self.get_bip9_status(bipName)['since'], 432) + assert_equal(self.get_bip9_status(bipName)['since'], 576) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) @@ -142,7 +185,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') - assert_equal(self.get_bip9_status(bipName)['since'], 432) + assert_equal(self.get_bip9_status(bipName)['since'], 576) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) @@ -168,7 +211,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance([[block, True]]) assert_equal(self.get_bip9_status(bipName)['status'], 'active') - assert_equal(self.get_bip9_status(bipName)['since'], 576) + assert_equal(self.get_bip9_status(bipName)['since'], 720) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName in tmpl['rules']) assert(bipName not in tmpl['vbavailable']) @@ -196,7 +239,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): # Restart all self.test.clear_all_connections() - stop_nodes(self.nodes) + self.stop_nodes() shutil.rmtree(self.options.tmpdir + "/node0") self.setup_chain() self.setup_network() diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 8596f1edaf..6aef6d4489 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -10,6 +10,7 @@ Test the following RPCs: - getbestblockhash - getblockhash - getblockheader + - getchaintxstats - getnetworkhashps - verifychain @@ -32,15 +33,24 @@ class BlockchainTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = False - self.num_nodes = 2 + self.num_nodes = 1 def run_test(self): + self._test_getchaintxstats() self._test_gettxoutsetinfo() self._test_getblockheader() self._test_getdifficulty() self._test_getnetworkhashps() self.nodes[0].verifychain(4, 0) + def _test_getchaintxstats(self): + chaintxstats = self.nodes[0].getchaintxstats(1) + # 200 txs plus genesis tx + assert_equal(chaintxstats['txcount'], 201) + # tx rate should be 1 per 10 minutes, or 1/600 + # we have to round because of binary math + assert_equal(round(chaintxstats['txrate'] * 600, 10), Decimal(1)) + def _test_gettxoutsetinfo(self): node = self.nodes[0] res = node.gettxoutsetinfo() @@ -49,9 +59,38 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res['transactions'], 200) assert_equal(res['height'], 200) assert_equal(res['txouts'], 200) - assert_equal(res['bytes_serialized'], 13924), + assert_equal(res['bogosize'], 17000), + assert_equal(res['bestblock'], node.getblockhash(200)) + size = res['disk_size'] + assert size > 6400 + assert size < 64000 assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized']), 64) + assert_equal(len(res['hash_serialized_2']), 64) + + self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block") + b1hash = node.getblockhash(1) + node.invalidateblock(b1hash) + + res2 = node.gettxoutsetinfo() + assert_equal(res2['transactions'], 0) + assert_equal(res2['total_amount'], Decimal('0')) + assert_equal(res2['height'], 0) + assert_equal(res2['txouts'], 0) + assert_equal(res2['bogosize'], 0), + assert_equal(res2['bestblock'], node.getblockhash(0)) + assert_equal(len(res2['hash_serialized_2']), 64) + + self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") + node.reconsiderblock(b1hash) + + res3 = node.gettxoutsetinfo() + assert_equal(res['total_amount'], res3['total_amount']) + assert_equal(res['transactions'], res3['transactions']) + assert_equal(res['height'], res3['height']) + assert_equal(res['txouts'], res3['txouts']) + assert_equal(res['bogosize'], res3['bogosize']) + assert_equal(res['bestblock'], res3['bestblock']) + assert_equal(res['hash_serialized_2'], res3['hash_serialized_2']) def _test_getblockheader(self): node = self.nodes[0] diff --git a/test/functional/bumpfee.py b/test/functional/bumpfee.py index 54fd7740c1..d42bab6cbf 100755 --- a/test/functional/bumpfee.py +++ b/test/functional/bumpfee.py @@ -38,12 +38,12 @@ class BumpFeeTest(BitcoinTestFramework): def setup_network(self, split=False): extra_args = [["-prematurewitness", "-walletprematurewitness", "-walletrbf={}".format(i)] for i in range(self.num_nodes)] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) # Encrypt wallet for test_locked_wallet_fails test self.nodes[1].encryptwallet(WALLET_PASSPHRASE) bitcoind_processes[1].wait() - self.nodes[1] = start_node(1, self.options.tmpdir, extra_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, extra_args[1]) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) connect_nodes_bi(self.nodes, 0, 1) @@ -88,6 +88,7 @@ def test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address): sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() bumped_tx = rbf_node.bumpfee(rbfid) + assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 # check that bumped_tx propogates, original tx was evicted and has a wallet conflict sync_mempools((rbf_node, peer_node)) diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py index f453fc0261..89b68aeb25 100755 --- a/test/functional/disconnect_ban.py +++ b/test/functional/disconnect_ban.py @@ -9,10 +9,7 @@ from test_framework.mininode import wait_until from test_framework.test_framework import BitcoinTestFramework from test_framework.util import (assert_equal, assert_raises_jsonrpc, - connect_nodes_bi, - start_node, - stop_node, - ) + connect_nodes_bi) class DisconnectBanTest(BitcoinTestFramework): @@ -68,9 +65,9 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setmocktime(old_time + 3) assert_equal(len(self.nodes[1].listbanned()), 3) - stop_node(self.nodes[1], 1) + self.stop_node(1) - self.nodes[1] = start_node(1, self.options.tmpdir) + self.nodes[1] = self.start_node(1, self.options.tmpdir) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) diff --git a/test/functional/forknotify.py b/test/functional/forknotify.py index 9db61c8350..3bcf0a6795 100755 --- a/test/functional/forknotify.py +++ b/test/functional/forknotify.py @@ -21,10 +21,10 @@ class ForkNotifyTest(BitcoinTestFramework): self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") with open(self.alert_filename, 'w', encoding='utf8'): pass # Just open then close to create zero-length file - self.nodes.append(start_node(0, self.options.tmpdir, + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-blockversion=2", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""])) # Node1 mines block.version=211 blocks - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-blockversion=211"])) connect_nodes(self.nodes[1], 0) diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index 9ddafeb611..0a3166b89b 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -448,13 +448,13 @@ class RawTransactionsTest(BitcoinTestFramework): ############################################################ # locked wallet test + self.stop_node(0) + self.stop_node(2) + self.stop_node(3) self.nodes[1].encryptwallet("test") - self.nodes.pop(1) - stop_node(self.nodes[0], 0) - stop_node(self.nodes[1], 2) - stop_node(self.nodes[2], 3) + bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate for node in self.nodes: diff --git a/test/functional/import-rescan.py b/test/functional/import-rescan.py index 5be095e62d..4fc5078217 100755 --- a/test/functional/import-rescan.py +++ b/test/functional/import-rescan.py @@ -21,7 +21,7 @@ happened previously. from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import (connect_nodes, sync_blocks, assert_equal, set_node_times) import collections import enum @@ -121,7 +121,7 @@ class ImportRescanTest(BitcoinTestFramework): if import_node.prune: extra_args[i] += ["-prune=1"] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) for i in range(1, self.num_nodes): connect_nodes(self.nodes[i], 0) diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index 9e3491c428..e83e85de13 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -428,8 +428,8 @@ class ImportMultiTest (BitcoinTestFramework): # restart nodes to check for proper serialization/deserialization of watch only address - stop_nodes(self.nodes) - self.nodes = start_nodes(2, self.options.tmpdir) + self.stop_nodes() + self.nodes = self.start_nodes(2, self.options.tmpdir) address_assert = self.nodes[1].validateaddress(watchonly_address) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) diff --git a/test/functional/keypool.py b/test/functional/keypool.py index c276e64c7c..f23a427d1f 100755 --- a/test/functional/keypool.py +++ b/test/functional/keypool.py @@ -20,7 +20,7 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].encryptwallet('test') bitcoind_processes[0].wait() # Restart node 0 - nodes[0] = start_node(0, self.options.tmpdir) + nodes[0] = self.start_node(0, self.options.tmpdir) # Keep creating keys addr = nodes[0].getnewaddress() addr_data = nodes[0].validateaddress(addr) diff --git a/test/functional/listtransactions.py b/test/functional/listtransactions.py index cba370d8b0..f69f1c5724 100755 --- a/test/functional/listtransactions.py +++ b/test/functional/listtransactions.py @@ -24,7 +24,7 @@ class ListTransactionsTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime enable_mocktime() - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): # Simple send, 0 to 1: diff --git a/test/functional/maxuploadtarget.py b/test/functional/maxuploadtarget.py index bff1b53234..66e5bd29e6 100755 --- a/test/functional/maxuploadtarget.py +++ b/test/functional/maxuploadtarget.py @@ -146,8 +146,8 @@ class MaxUploadTest(BitcoinTestFramework): #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 self.log.info("Restarting nodes with -whitelist=127.0.0.1") - stop_node(self.nodes[0], 0) - self.nodes[0] = start_node(0, self.options.tmpdir, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) #recreate/reconnect a test node test_nodes = [TestNode()] diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 7b15476ea2..e0889fd5e9 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -63,27 +63,27 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[1].getrawmempool()), 5) self.log.debug("Stop-start node0 and node1. Verify that node0 has the transactions in its mempool and node1 does not.") - stop_nodes(self.nodes) + self.stop_nodes() self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir)) - self.nodes.append(start_node(1, self.options.tmpdir)) + self.nodes.append(self.start_node(0, self.options.tmpdir)) + self.nodes.append(self.start_node(1, self.options.tmpdir)) # Give bitcoind a second to reload the mempool time.sleep(1) assert wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) assert_equal(len(self.nodes[1].getrawmempool()), 0) self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") - stop_nodes(self.nodes) + self.stop_nodes() self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-persistmempool=0"])) + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-persistmempool=0"])) # Give bitcoind a second to reload the mempool time.sleep(1) assert_equal(len(self.nodes[0].getrawmempool()), 0) self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") - stop_nodes(self.nodes) + self.stop_nodes() self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir)) + self.nodes.append(self.start_node(0, self.options.tmpdir)) assert wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) if __name__ == '__main__': diff --git a/test/functional/net.py b/test/functional/net.py index fb46064441..3ba3764cf9 100755 --- a/test/functional/net.py +++ b/test/functional/net.py @@ -29,6 +29,7 @@ class NetTest(BitcoinTestFramework): self._test_getnettotals() self._test_getnetworkinginfo() self._test_getaddednodeinfo() + self._test_getpeerinfo() def _test_connection_count(self): # connect_nodes_bi connects each node to the other @@ -88,6 +89,12 @@ class NetTest(BitcoinTestFramework): assert_raises_jsonrpc(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') + def _test_getpeerinfo(self): + peer_info = [x.getpeerinfo() for x in self.nodes] + # check both sides of bidirectional connection between nodes + # the address bound to on one side will be the source address for the other node + assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) + assert_equal(peer_info[1][0]['addrbind'], peer_info[0][0]['addr']) if __name__ == '__main__': NetTest().main() diff --git a/test/functional/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py index 9b302120ac..ff76e49fba 100755 --- a/test/functional/p2p-compactblocks.py +++ b/test/functional/p2p-compactblocks.py @@ -98,7 +98,7 @@ class CompactBlocksTest(BitcoinTestFramework): self.setup_clean_chain = True # Node0 = pre-segwit, node1 = segwit-aware self.num_nodes = 2 - self.extra_args = [["-bip9params=segwit:0:0"], ["-txindex"]] + self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]] self.utxos = [] def build_block_on_tip(self, node, segwit=False): diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py index 24d4d37c42..dbc61d21fc 100755 --- a/test/functional/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -114,7 +114,7 @@ class SegWitTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-whitelist=127.0.0.1"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"], ["-whitelist=127.0.0.1", "-bip9params=segwit:0:0"]] + self.extra_args = [["-whitelist=127.0.0.1"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] def setup_network(self): self.setup_nodes() @@ -1495,8 +1495,8 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # Restart with the new binary - stop_node(node, node_id) - self.nodes[node_id] = start_node(node_id, self.options.tmpdir) + self.stop_node(node_id) + self.nodes[node_id] = self.start_node(node_id, self.options.tmpdir) connect_nodes(self.nodes[0], node_id) sync_blocks(self.nodes) diff --git a/test/functional/p2p-versionbits-warning.py b/test/functional/p2p-versionbits-warning.py index 41921fe14e..df7e8ce5c1 100755 --- a/test/functional/p2p-versionbits-warning.py +++ b/test/functional/p2p-versionbits-warning.py @@ -108,22 +108,22 @@ class VersionBitsWarningTest(BitcoinTestFramework): # is cleared, and restart the node. This should move the versionbit state # to ACTIVE. self.nodes[0].generate(VB_PERIOD) - stop_nodes(self.nodes) + self.stop_nodes() # Empty out the alert file with open(self.alert_filename, 'w', encoding='utf8') as _: pass - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) # Connecting one block should be enough to generate an error. self.nodes[0].generate(1) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getinfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getmininginfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getnetworkinfo()["warnings"]) - stop_nodes(self.nodes) + self.stop_nodes() self.test_versionbits_in_alert_file() # Test framework expects the node to still be running... - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) if __name__ == '__main__': VersionBitsWarningTest().main() diff --git a/test/functional/proxy_test.py b/test/functional/proxy_test.py index 69384d9d85..ae6f843ddc 100755 --- a/test/functional/proxy_test.py +++ b/test/functional/proxy_test.py @@ -35,7 +35,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( PORT_MIN, PORT_RANGE, - start_nodes, assert_equal, ) from test_framework.netutil import test_ipv6_local @@ -90,7 +89,7 @@ class ProxyTest(BitcoinTestFramework): ] if self.have_ipv6: args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) def node_test(self, node, proxies, auth, test_onion=True): rv = [] diff --git a/test/functional/pruning.py b/test/functional/pruning.py index 17019c658b..4c3501ad77 100755 --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -34,10 +34,11 @@ class PruneTest(BitcoinTestFramework): # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. + self.full_node_default_args = ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000" ] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect - self.extra_args = [["-maxreceivebuffer=20000", "-blockmaxsize=999000", "-checkblocks=5"], - ["-maxreceivebuffer=20000", "-blockmaxsize=999000", "-checkblocks=5"], + self.extra_args = [self.full_node_default_args, + self.full_node_default_args, ["-maxreceivebuffer=20000", "-prune=550"], ["-maxreceivebuffer=20000", "-blockmaxsize=999000"], ["-maxreceivebuffer=20000", "-blockmaxsize=999000"], @@ -97,12 +98,15 @@ class PruneTest(BitcoinTestFramework): # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine self.stop_node(0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) + self.nodes[0]=self.start_node(0, self.options.tmpdir, self.full_node_default_args, timewait=900) # Mine 24 blocks in node 1 for i in range(24): if j == 0: mine_large_block(self.nodes[1], self.utxo_cache_1) else: + # Add node1's wallet transactions back to the mempool, to + # avoid the mined blocks from being too small. + self.nodes[1].resendwallettransactions() self.nodes[1].generate(1) #tx's already in mempool from previous disconnects # Reorg back with 25 block chain from node 0 @@ -122,7 +126,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) self.stop_node(1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) + self.nodes[1] = self.start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) @@ -145,7 +149,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node1 to clear those giant tx's from mempool self.stop_node(1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) + self.nodes[1] = self.start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) @@ -159,6 +163,11 @@ class PruneTest(BitcoinTestFramework): self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + + # Get node0's wallet transactions back in its mempool, to avoid the + # mined blocks from being too small. + self.nodes[0].resendwallettransactions() + for i in range(22): # This can be slow, so do this in multiple RPC calls to avoid # RPC timeouts. @@ -218,13 +227,13 @@ class PruneTest(BitcoinTestFramework): def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode - node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, timewait=900) + node = self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, timewait=900) assert_equal(node.getblockcount(), 995) assert_raises_jsonrpc(-1, "not in prune mode", node.pruneblockchain, 500) self.stop_node(node_number) # now re-start in manual pruning mode - node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-prune=1"], timewait=900) + node = self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, ["-prune=1"], timewait=900) assert_equal(node.getblockcount(), 995) def height(index): @@ -298,7 +307,7 @@ class PruneTest(BitcoinTestFramework): # stop node, start back up with auto-prune at 550MB, make sure still runs self.stop_node(node_number) - self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-prune=550"], timewait=900) + self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, ["-prune=550"], timewait=900) self.log.info("Success") @@ -306,7 +315,7 @@ class PruneTest(BitcoinTestFramework): # check that the pruning node's wallet is still in good shape self.log.info("Stop and start pruning node to trigger wallet rescan") self.stop_node(2) - start_node(2, self.options.tmpdir, ["-prune=550"]) + self.start_node(2, self.options.tmpdir, ["-prune=550"]) self.log.info("Success") # check that wallet loads loads successfully when restarting a pruned node after IBD. @@ -316,7 +325,7 @@ class PruneTest(BitcoinTestFramework): nds = [self.nodes[0], self.nodes[5]] sync_blocks(nds, wait=5, timeout=300) self.stop_node(5) #stop and start to trigger rescan - start_node(5, self.options.tmpdir, ["-prune=550"]) + self.start_node(5, self.options.tmpdir, ["-prune=550"]) self.log.info("Success") def run_test(self): diff --git a/test/functional/receivedby.py b/test/functional/receivedby.py index a1cae301c5..2cad6269ac 100755 --- a/test/functional/receivedby.py +++ b/test/functional/receivedby.py @@ -32,7 +32,7 @@ class ReceivedByTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime enable_mocktime() - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): ''' diff --git a/test/functional/reindex.py b/test/functional/reindex.py index 8b8c5f3e71..b446baa04d 100755 --- a/test/functional/reindex.py +++ b/test/functional/reindex.py @@ -10,11 +10,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - start_nodes, - stop_nodes, - assert_equal, -) +from test_framework.util import assert_equal import time class ReindexTest(BitcoinTestFramework): @@ -27,9 +23,9 @@ class ReindexTest(BitcoinTestFramework): def reindex(self, justchainstate=False): self.nodes[0].generate(3) blockcount = self.nodes[0].getblockcount() - stop_nodes(self.nodes) + self.stop_nodes() extra_args = [["-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) while self.nodes[0].getblockcount() < blockcount: time.sleep(0.1) assert_equal(self.nodes[0].getblockcount(), blockcount) diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index efc36481d1..5336cf2ec8 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -7,7 +7,7 @@ import socket import sys -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import * from test_framework.netutil import * @@ -36,10 +36,10 @@ class RPCBindTest(BitcoinTestFramework): if allow_ips: base_args += ['-rpcallowip=' + x for x in allow_ips] binds = ['-rpcbind='+addr for addr in addresses] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) pid = bitcoind_processes[0].pid assert_equal(set(get_bind_addrs(pid)), set(expected)) - stop_nodes(self.nodes) + self.stop_nodes() def run_allowip_test(self, allow_ips, rpchost, rpcport): ''' @@ -47,17 +47,16 @@ class RPCBindTest(BitcoinTestFramework): at a non-localhost IP. ''' base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) # connect to node through non-loopback interface node = get_rpc_proxy(rpc_url(0, "%s:%d" % (rpchost, rpcport)), 0) node.getnetworkinfo() - stop_nodes(self.nodes) + self.stop_nodes() def run_test(self): # due to OS-specific network stats queries, this test works only on Linux if not sys.platform.startswith('linux'): - self.log.warning("This test can only be run on linux. Skipping test.") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("This test can only be run on linux.") # find the first non-loopback interface for testing non_loopback_ip = None for name,ip in all_interfaces(): @@ -65,15 +64,13 @@ class RPCBindTest(BitcoinTestFramework): non_loopback_ip = ip break if non_loopback_ip is None: - self.log.warning("This test requires at least one non-loopback IPv4 interface. Skipping test.") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("This test requires at least one non-loopback IPv4 interface.") try: s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) s.connect(("::1",1)) s.close except OSError: - self.log.warning("This test requires IPv6 support. Skipping test.") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("This test requires IPv6 support.") self.log.info("Using interface %s for testing" % non_loopback_ip) diff --git a/test/functional/smartfees.py b/test/functional/smartfees.py index 4124f8025e..482c77863f 100755 --- a/test/functional/smartfees.py +++ b/test/functional/smartfees.py @@ -155,7 +155,7 @@ class EstimateFeeTest(BitcoinTestFramework): """ self.nodes = [] # Use node0 to mine blocks for input splitting - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-whitelist=127.0.0.1"])) self.log.info("This test is time consuming, please be patient") @@ -191,7 +191,7 @@ class EstimateFeeTest(BitcoinTestFramework): # Node1 mines small blocks but that are bigger than the expected transaction rate. # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, # (17k is room enough for 110 or so transactions) - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-blockmaxsize=17000", "-maxorphantx=1000"])) connect_nodes(self.nodes[1], 0) @@ -199,7 +199,7 @@ class EstimateFeeTest(BitcoinTestFramework): # produces too small blocks (room for only 55 or so transactions) node2args = ["-blockmaxsize=8000", "-maxorphantx=1000"] - self.nodes.append(start_node(2, self.options.tmpdir, node2args)) + self.nodes.append(self.start_node(2, self.options.tmpdir, node2args)) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[2], 1) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8d8139e4e4..67abf35687 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -5,6 +5,7 @@ """Base class for RPC testing.""" from collections import deque +from enum import Enum import logging import optparse import os @@ -31,16 +32,25 @@ from .util import ( p2p_port, rpc_url, set_node_times, - start_node, - start_nodes, - stop_node, - stop_nodes, + _start_node, + _start_nodes, + _stop_node, + _stop_nodes, sync_blocks, sync_mempools, wait_for_bitcoind_start, ) from .authproxy import JSONRPCException +class TestStatus(Enum): + PASSED = 1 + FAILED = 2 + SKIPPED = 3 + +TEST_EXIT_PASSED = 0 +TEST_EXIT_FAILED = 1 +TEST_EXIT_SKIPPED = 77 + class BitcoinTestFramework(object): """Base class for a bitcoin test script. @@ -57,11 +67,6 @@ class BitcoinTestFramework(object): This class also contains various public and private helper methods.""" # Methods to override in subclass test scripts. - - TEST_EXIT_PASSED = 0 - TEST_EXIT_FAILED = 1 - TEST_EXIT_SKIPPED = 77 - def __init__(self): self.num_nodes = 4 self.setup_clean_chain = False @@ -91,7 +96,7 @@ class BitcoinTestFramework(object): extra_args = None if hasattr(self, "extra_args"): extra_args = self.extra_args - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = _start_nodes(self.num_nodes, self.options.tmpdir, extra_args) def run_test(self): raise NotImplementedError @@ -109,8 +114,7 @@ class BitcoinTestFramework(object): help="Source directory containing bitcoind/bitcoin-cli (default: %default)") parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"), help="Directory for caching pregenerated datadirs") - parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), - help="Root directory for datadirs") + parser.add_option("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", @@ -124,9 +128,6 @@ class BitcoinTestFramework(object): self.add_options(parser) (self.options, self.args) = parser.parse_args() - # backup dir variable for removal at cleanup - self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed) - if self.options.coveragedir: enable_coverage(self.options.coveragedir) @@ -137,18 +138,24 @@ class BitcoinTestFramework(object): check_json_precision() # Set up temp directory and start logging - os.makedirs(self.options.tmpdir, exist_ok=False) + if self.options.tmpdir: + os.makedirs(self.options.tmpdir, exist_ok=False) + else: + self.options.tmpdir = tempfile.mkdtemp(prefix="test") self._start_logging() - success = False + success = TestStatus.FAILED try: self.setup_chain() self.setup_network() self.run_test() - success = True + success = TestStatus.PASSED except JSONRPCException as e: self.log.exception("JSONRPC error") + except SkipTest as e: + self.log.warning("Test Skipped: %s" % e.message) + success = TestStatus.SKIPPED except AssertionError as e: self.log.exception("Assertion failed") except KeyError as e: @@ -160,15 +167,14 @@ class BitcoinTestFramework(object): if not self.options.noshutdown: self.log.info("Stopping nodes") - self.stop_nodes() + if self.nodes: + self.stop_nodes() else: self.log.info("Note: bitcoinds were not stopped and may still be running") - if not self.options.nocleanup and not self.options.noshutdown and success: + if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: self.log.info("Cleaning up") shutil.rmtree(self.options.tmpdir) - if not os.listdir(self.options.root): - os.rmdir(self.options.root) else: self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) if os.getenv("PYTHON_DEBUG", ""): @@ -186,27 +192,31 @@ class BitcoinTestFramework(object): except OSError: print("Opening file %s failed." % fn) traceback.print_exc() - if success: + + if success == TestStatus.PASSED: self.log.info("Tests successful") - sys.exit(self.TEST_EXIT_PASSED) + sys.exit(TEST_EXIT_PASSED) + elif success == TestStatus.SKIPPED: + self.log.info("Test skipped") + sys.exit(TEST_EXIT_SKIPPED) else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) logging.shutdown() - sys.exit(self.TEST_EXIT_FAILED) + sys.exit(TEST_EXIT_FAILED) # Public helper methods. These can be accessed by the subclass test scripts. def start_node(self, i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): - return start_node(i, dirname, extra_args, rpchost, timewait, binary, stderr) + return _start_node(i, dirname, extra_args, rpchost, timewait, binary, stderr) def start_nodes(self, num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - return start_nodes(num_nodes, dirname, extra_args, rpchost, timewait, binary) + return _start_nodes(num_nodes, dirname, extra_args, rpchost, timewait, binary) def stop_node(self, num_node): - stop_node(self.nodes[num_node], num_node) + _stop_node(self.nodes[num_node], num_node) def stop_nodes(self): - stop_nodes(self.nodes) + _stop_nodes(self.nodes) def split_network(self): """ @@ -349,6 +359,11 @@ class BitcoinTestFramework(object): # 2 binaries: 1 test binary, 1 ref binary # n>2 binaries: 1 test binary, n-1 ref binaries +class SkipTest(Exception): + """This exception is raised to skip a test""" + def __init__(self, message): + self.message = message + class ComparisonTestFramework(BitcoinTestFramework): def __init__(self): diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 2b56fe8d62..2b0f32c2b6 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -227,10 +227,11 @@ def wait_for_bitcoind_start(process, url, i): time.sleep(0.25) -def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): - """ - Start a bitcoind and return RPC connection to it - """ +def _start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): + """Start a bitcoind and return RPC connection to it + + This function should only be called from within test_framework, not by individual test scripts.""" + datadir = os.path.join(dirname, "node"+str(i)) if binary is None: binary = os.getenv("BITCOIND", "bitcoind") @@ -251,8 +252,8 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: try: - node = start_node(i, dirname, extra_args, stderr=log_stderr) - stop_node(node, i) + node = _start_node(i, dirname, extra_args, stderr=log_stderr) + _stop_node(node, i) except Exception as e: assert 'bitcoind exited' in str(e) #node must have shutdown if expected_msg is not None: @@ -267,10 +268,11 @@ def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=Non assert_msg = "bitcoind should have exited with expected error " + expected_msg raise AssertionError(assert_msg) -def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - """ - Start multiple bitcoinds, return RPC connections to them - """ +def _start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): + """Start multiple bitcoinds, return RPC connections to them + + This function should only be called from within test_framework, not by individual test scripts.""" + if extra_args is None: extra_args = [ None for _ in range(num_nodes) ] if binary is None: binary = [ None for _ in range(num_nodes) ] assert_equal(len(extra_args), num_nodes) @@ -278,16 +280,20 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None rpcs = [] try: for i in range(num_nodes): - rpcs.append(start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) + rpcs.append(_start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) except: # If one node failed to start, stop the others - stop_nodes(rpcs) + _stop_nodes(rpcs) raise return rpcs def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node"+str(n_node), "regtest", logname) -def stop_node(node, i): +def _stop_node(node, i): + """Stop a bitcoind test node + + This function should only be called from within test_framework, not by individual test scripts.""" + logger.debug("Stopping node %d" % i) try: node.stop() @@ -297,9 +303,13 @@ def stop_node(node, i): assert_equal(return_code, 0) del bitcoind_processes[i] -def stop_nodes(nodes): +def _stop_nodes(nodes): + """Stop multiple bitcoind test nodes + + This function should only be called from within test_framework, not by individual test scripts.""" + for i, node in enumerate(nodes): - stop_node(node, i) + _stop_node(node, i) assert not bitcoind_processes.values() # All connections must be gone now def set_node_times(nodes, t): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6174ca1d88..b2aee7c739 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -16,6 +16,7 @@ For a description of arguments recognized by test scripts, see import argparse import configparser +import datetime import os import time import shutil @@ -170,6 +171,7 @@ def main(): parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') + parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") args, unknown_args = parser.parse_known_args() # args to be passed on always start with two dashes; tests are the remaining unknown args @@ -187,6 +189,12 @@ def main(): logging_level = logging.INFO if args.quiet else logging.DEBUG logging.basicConfig(format='%(message)s', level=logging_level) + # Create base test directory + tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) + os.makedirs(tmpdir) + + logging.debug("Temporary test directory at %s" % tmpdir) + enable_wallet = config["components"].getboolean("ENABLE_WALLET") enable_utils = config["components"].getboolean("ENABLE_UTILS") enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") @@ -247,9 +255,9 @@ def main(): if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) -def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): +def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): # Warn if bitcoind is already running (unix only) try: if subprocess.check_output(["pidof", "bitcoind"]) is not None: @@ -280,10 +288,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([tests_dir + 'create_cache.py'] + flags) + subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) #Run Tests - job_queue = TestHandler(jobs, tests_dir, test_list, flags) + job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) time0 = time.time() test_results = [] @@ -310,6 +318,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal logging.debug("Cleaning up coverage data") coverage.cleanup() + # Clear up the temp directory if all subdirectories are gone + if not os.listdir(tmpdir): + os.rmdir(tmpdir) + all_passed = all(map(lambda test_result: test_result.was_successful, test_results)) sys.exit(not all_passed) @@ -337,10 +349,11 @@ class TestHandler: Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None): + def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): assert(num_tests_parallel >= 1) self.num_jobs = num_tests_parallel self.tests_dir = tests_dir + self.tmpdir = tmpdir self.test_list = test_list self.flags = flags self.num_running = 0 @@ -355,13 +368,15 @@ class TestHandler: # Add tests self.num_running += 1 t = self.test_list.pop(0) - port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)] + portseed = len(self.test_list) + self.portseed_offset + portseed_arg = ["--portseed={}".format(portseed)] log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) test_argv = t.split() + tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)] self.jobs.append((t, time.time(), - subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + port_seed, + subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), diff --git a/test/functional/wallet-accounts.py b/test/functional/wallet-accounts.py index e6635bea1c..158aa9ae89 100755 --- a/test/functional/wallet-accounts.py +++ b/test/functional/wallet-accounts.py @@ -14,9 +14,7 @@ RPCs tested are: """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, -) +from test_framework.util import assert_equal class WalletAccountsTest(BitcoinTestFramework): diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 8876f935a4..9cb32d4650 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -4,8 +4,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" +import os + from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, start_node, assert_equal, bitcoind_processes) +from test_framework.util import (assert_equal, bitcoind_processes) def read_dump(file_name, addrs, hd_master_addr_old): @@ -66,7 +68,7 @@ class WalletDumpTest(BitcoinTestFramework): # longer than the default 30 seconds due to an expensive # CWallet::TopUpKeyPool call, and the encryptwallet RPC made later in # the test often takes even longer. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) def run_test (self): tmpdir = self.options.tmpdir @@ -82,7 +84,8 @@ class WalletDumpTest(BitcoinTestFramework): self.nodes[0].keypoolrefill() # dump unencrypted wallet - self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") + result = self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") + assert_equal(result['filename'], os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump")) found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) @@ -93,7 +96,7 @@ class WalletDumpTest(BitcoinTestFramework): #encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') bitcoind_processes[0].wait() - self.nodes[0] = start_node(0, self.options.tmpdir, self.extra_args[0]) + self.nodes[0] = self.start_node(0, self.options.tmpdir, self.extra_args[0]) self.nodes[0].walletpassphrase('test', 10) # Should be a no-op: self.nodes[0].keypoolrefill() diff --git a/test/functional/wallet-hd.py b/test/functional/wallet-hd.py index aab3b4bc2d..e7ec72a248 100755 --- a/test/functional/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -6,7 +6,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - start_node, assert_equal, connect_nodes_bi, assert_start_raises_init_error @@ -29,7 +28,7 @@ class WalletHDTest(BitcoinTestFramework): # Make sure can't switch off usehd after wallet creation self.stop_node(1) assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet') - self.nodes[1] = start_node(1, self.options.tmpdir, self.extra_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) # Make sure we use hd, keep masterkeyid @@ -76,7 +75,7 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat") - self.nodes[1] = start_node(1, self.options.tmpdir, self.extra_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) #connect_nodes_bi(self.nodes, 0, 1) # Assert that derivation is deterministic @@ -90,7 +89,7 @@ class WalletHDTest(BitcoinTestFramework): # Needs rescan self.stop_node(1) - self.nodes[1] = start_node(1, self.options.tmpdir, self.extra_args[1] + ['-rescan']) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1] + ['-rescan']) #connect_nodes_bi(self.nodes, 0, 1) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) diff --git a/test/functional/wallet.py b/test/functional/wallet.py index efde2e005e..3e3e8fcddb 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -21,7 +21,7 @@ class WalletTest(BitcoinTestFramework): self.extra_args = [['-usehd={:d}'.format(i%2==0)] for i in range(4)] def setup_network(self): - self.nodes = start_nodes(3, self.options.tmpdir, self.extra_args[:3]) + self.nodes = self.start_nodes(3, self.options.tmpdir, self.extra_args[:3]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -178,7 +178,7 @@ class WalletTest(BitcoinTestFramework): txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) sync_mempools(self.nodes) - self.nodes.append(start_node(3, self.options.tmpdir, self.extra_args[3])) + self.nodes.append(self.start_node(3, self.options.tmpdir, self.extra_args[3])) connect_nodes_bi(self.nodes, 0, 3) sync_blocks(self.nodes) @@ -221,8 +221,8 @@ class WalletTest(BitcoinTestFramework): assert(found) #do some -walletbroadcast tests - stop_nodes(self.nodes) - self.nodes = start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) + self.stop_nodes() + self.nodes = self.start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -246,8 +246,8 @@ class WalletTest(BitcoinTestFramework): txIdNotBroadcasted = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) #restart the nodes with -walletbroadcast=1 - stop_nodes(self.nodes) - self.nodes = start_nodes(3, self.options.tmpdir) + self.stop_nodes() + self.nodes = self.start_nodes(3, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -348,9 +348,9 @@ class WalletTest(BitcoinTestFramework): chainlimit = 6 for m in maintenance: self.log.info("check " + m) - stop_nodes(self.nodes) + self.stop_nodes() # set lower ancestor limit for later - self.nodes = start_nodes(3, self.options.tmpdir, [[m, "-limitancestorcount="+str(chainlimit)]] * 3) + self.nodes = self.start_nodes(3, self.options.tmpdir, [[m, "-limitancestorcount="+str(chainlimit)]] * 3) while m == '-reindex' and [block_count] * 3 != [self.nodes[i].getblockcount() for i in range(3)]: # reindex will leave rpc warm up "early"; Wait for it to finish time.sleep(0.1) @@ -397,8 +397,8 @@ class WalletTest(BitcoinTestFramework): # Try with walletrejectlongchains # Double chain limit but require combining inputs, so we pass SelectCoinsMinConf - stop_node(self.nodes[0],0) - self.nodes[0] = start_node(0, self.options.tmpdir, ["-walletrejectlongchains", "-limitancestorcount="+str(2*chainlimit)]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-walletrejectlongchains", "-limitancestorcount="+str(2*chainlimit)]) # wait for loadmempool timeout = 10 diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 0492132af6..a4507182a2 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -77,18 +77,18 @@ class WalletBackupTest(BitcoinTestFramework): # As above, this mirrors the original bash test. def start_three(self): - self.nodes[0] = start_node(0, self.options.tmpdir) - self.nodes[1] = start_node(1, self.options.tmpdir) - self.nodes[2] = start_node(2, self.options.tmpdir) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + self.nodes[1] = self.start_node(1, self.options.tmpdir) + self.nodes[2] = self.start_node(2, self.options.tmpdir) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) connect_nodes(self.nodes[2], 3) connect_nodes(self.nodes[2], 0) def stop_three(self): - stop_node(self.nodes[0], 0) - stop_node(self.nodes[1], 1) - stop_node(self.nodes[2], 2) + self.stop_node(0) + self.stop_node(1) + self.stop_node(2) def erase_three(self): os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 7987edeb54..a8600e82f6 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -58,18 +58,16 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(tx3['txid'], txid3) #tx3 must be available (unconfirmed) #restart bitcoind - self.nodes[0].stop() - bitcoind_processes[0].wait() - self.nodes[0] = start_node(0,self.options.tmpdir) + self.stop_node(0) + self.nodes[0] = self.start_node(0,self.options.tmpdir) tx3 = self.nodes[0].gettransaction(txid3) assert_equal(tx3['txid'], txid3) #tx must be available (unconfirmed) - self.nodes[0].stop() - bitcoind_processes[0].wait() + self.stop_node(0) #restart bitcoind with zapwallettxes - self.nodes[0] = start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) + self.nodes[0] = self.start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) #there must be a expection because the unconfirmed wallettx0 must be gone by now diff --git a/test/functional/zmq_test.py b/test/functional/zmq_test.py index 918e13bcd4..ce39cfefdc 100755 --- a/test/functional/zmq_test.py +++ b/test/functional/zmq_test.py @@ -6,9 +6,8 @@ import configparser import os import struct -import sys -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import * class ZMQTest (BitcoinTestFramework): @@ -24,8 +23,7 @@ class ZMQTest (BitcoinTestFramework): try: import zmq except ImportError: - self.log.warning("python3-zmq module not available. Skipping zmq tests!") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("python3-zmq module not available.") # Check that bitcoin has been built with ZMQ enabled config = configparser.ConfigParser() @@ -34,15 +32,14 @@ class ZMQTest (BitcoinTestFramework): config.read_file(open(self.options.configfile)) if not config["components"].getboolean("ENABLE_ZMQ"): - self.log.warning("bitcoind has not been built with zmq enabled. Skipping zmq tests!") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("bitcoind has not been built with zmq enabled.") self.zmqContext = zmq.Context() self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port) - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)], [], [], |