diff options
42 files changed, 868 insertions, 130 deletions
diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 0000000000..dee25ff67b --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>' + echo + echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/depends/README.md b/depends/README.md index aaa062774f..e5e2a8a653 100644 --- a/depends/README.md +++ b/depends/README.md @@ -30,17 +30,12 @@ Common `host-platform-triplets` for cross compilation are: - `aarch64-linux-gnu` for Linux ARM 64 bit - `riscv32-linux-gnu` for Linux RISC-V 32 bit - `riscv64-linux-gnu` for Linux RISC-V 64 bit +- `armv7a-linux-android` for Android ARM 32 bit - `aarch64-linux-android` for Android ARM 64 bit +- `i686-linux-android` for Android x86 32 bit +- `x86_64-linux-android` for Android x86 64 bit -The paths are automatically configured and no other options are needed unless targeting Android. -Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). -In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. -If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. -This is an example command for a default build with no disabled dependencies: - - ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin - - +The paths are automatically configured and no other options are needed unless targeting [Android](#Android). ### Install the required dependencies: Ubuntu & Debian @@ -99,6 +94,19 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-win: run 'make download-win' to fetch all sources needed for win builds download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Android + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 1756648157..e7fd8102a4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -384,7 +384,7 @@ Threads - ThreadScriptCheck : Verifies block scripts. -- ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat. +- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`. - StartNode : Starts other threads. diff --git a/doc/release-notes-15954.md b/doc/release-notes-15954.md new file mode 100644 index 0000000000..f4d2c5688c --- /dev/null +++ b/doc/release-notes-15954.md @@ -0,0 +1,4 @@ +Configuration option changes +----------------------------- + +Importing blocks upon startup via the `bootstrap.dat` file no longer occurs by default. The file must now be specified with `-loadblock=<file>`. diff --git a/doc/release-notes.md b/doc/release-notes.md index ea82962e75..a47c8802b0 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -85,6 +85,7 @@ Wallet ------ - The wallet now by default uses bech32 addresses when using RPC, and creates native segwit change outputs. +- The way that output trust was computed has been fixed in #16766, which impacts confirmed/unconfirmed balance status and coin selection. Low-level changes ================= diff --git a/src/Makefile.am b/src/Makefile.am index 507e5cbb9f..ff4f071a3c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -146,6 +146,7 @@ BITCOIN_CORE_H = \ dbwrapper.h \ limitedmap.h \ logging.h \ + logging/timer.h \ memusage.h \ merkleblock.h \ miner.h \ @@ -161,6 +162,7 @@ BITCOIN_CORE_H = \ node/context.h \ node/psbt.h \ node/transaction.h \ + node/utxo_snapshot.h \ noui.h \ optional.h \ outputtype.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a4cf78d61b..742530c55c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -7,6 +7,7 @@ FUZZ_TARGETS = \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ test/fuzz/banentry_deserialize \ + test/fuzz/bech32 \ test/fuzz/block_deserialize \ test/fuzz/blockheader_deserialize \ test/fuzz/blocklocator_deserialize \ @@ -58,18 +59,24 @@ GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.r BITCOIN_TEST_SUITE = \ test/lib/blockfilter.cpp \ test/lib/blockfilter.h \ + test/lib/logging.cpp \ + test/lib/logging.h \ test/lib/transaction_utils.cpp \ test/lib/transaction_utils.h \ test/main.cpp \ test/setup_common.h \ - test/setup_common.cpp + test/setup_common.cpp \ + test/util/str.h \ + test/util/str.cpp FUZZ_SUITE = \ - test/setup_common.h \ - test/setup_common.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h \ - test/fuzz/FuzzedDataProvider.h + test/fuzz/FuzzedDataProvider.h \ + test/setup_common.cpp \ + test/setup_common.h \ + test/util/str.cpp \ + test/util/str.h FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_SERVER) \ @@ -120,6 +127,7 @@ BITCOIN_TESTS =\ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/logging_tests.cpp \ test/dbwrapper_tests.cpp \ test/validation_tests.cpp \ test/mempool_tests.cpp \ @@ -242,6 +250,12 @@ test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bech32_SOURCES = $(FUZZ_SUITE) test/fuzz/bech32.cpp +test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) + test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/init.cpp b/src/init.cpp index 1a99ca9abc..3eb8fc7976 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -372,7 +372,7 @@ void SetupServerArgs() gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -684,20 +684,6 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) LoadGenesisBlock(chainparams); } - // hardcoded $DATADIR/bootstrap.dat - fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (fs::exists(pathBootstrap)) { - FILE *file = fsbridge::fopen(pathBootstrap, "rb"); - if (file) { - fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; - LogPrintf("Importing bootstrap.dat...\n"); - LoadExternalBlockFile(chainparams, file); - RenameOver(pathBootstrap, pathBootstrapOld); - } else { - LogPrintf("Warning: Could not open bootstrap file %s\n", pathBootstrap.string()); - } - } - // -loadblock= for (const fs::path& path : vImportFiles) { FILE *file = fsbridge::fopen(path, "rb"); diff --git a/src/logging.cpp b/src/logging.cpp index 60ab486198..b01177f23f 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -67,6 +67,9 @@ bool BCLog::Logger::StartLogging() if (m_print_to_file) FileWriteStr(s, m_fileout); if (m_print_to_console) fwrite(s.data(), 1, s.size(), stdout); + for (const auto& cb : m_print_callbacks) { + cb(s); + } m_msgs_before_open.pop_front(); } @@ -81,6 +84,7 @@ void BCLog::Logger::DisconnectTestLogger() m_buffering = true; if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; + m_print_callbacks.clear(); } void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) @@ -270,6 +274,9 @@ void BCLog::Logger::LogPrintStr(const std::string& str) fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } + for (const auto& cb : m_print_callbacks) { + cb(str_prefixed); + } if (m_print_to_file) { assert(m_fileout != nullptr); diff --git a/src/logging.h b/src/logging.h index e37c0c823b..9ed41c2b98 100644 --- a/src/logging.h +++ b/src/logging.h @@ -77,6 +77,9 @@ namespace BCLog { std::string LogTimestampStr(const std::string& str); + /** Slots that connect to the print signal */ + std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {}; + public: bool m_print_to_console = false; bool m_print_to_file = false; @@ -95,7 +98,22 @@ namespace BCLog { bool Enabled() const { std::lock_guard<std::mutex> scoped_lock(m_cs); - return m_buffering || m_print_to_console || m_print_to_file; + return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); + } + + /** Connect a slot to the print signal and return the connection */ + std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.push_back(std::move(fun)); + return --m_print_callbacks.end(); + } + + /** Delete a connection */ + void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.erase(it); } /** Start logging (and flush all buffered messages) */ diff --git a/src/logging/timer.h b/src/logging/timer.h new file mode 100644 index 0000000000..34dbb942c5 --- /dev/null +++ b/src/logging/timer.h @@ -0,0 +1,104 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_LOGGING_TIMER_H +#define BITCOIN_LOGGING_TIMER_H + +#include <logging.h> +#include <util/macros.h> +#include <util/time.h> + +#include <chrono> +#include <string> + + +namespace BCLog { + +//! RAII-style object that outputs timing information to logs. +template <typename TimeType> +class Timer +{ +public: + //! If log_category is left as the default, end_msg will log unconditionally + //! (instead of being filtered by category). + Timer( + std::string prefix, + std::string end_msg, + BCLog::LogFlags log_category = BCLog::LogFlags::ALL) : + m_prefix(std::move(prefix)), + m_title(std::move(end_msg)), + m_log_category(log_category) + { + this->Log(strprintf("%s started", m_title)); + m_start_t = GetTime<std::chrono::microseconds>(); + } + + ~Timer() + { + this->Log(strprintf("%s completed", m_title)); + } + + void Log(const std::string& msg) + { + const std::string full_msg = this->LogMsg(msg); + + if (m_log_category == BCLog::LogFlags::ALL) { + LogPrintf("%s\n", full_msg); + } else { + LogPrint(m_log_category, "%s\n", full_msg); + } + } + + std::string LogMsg(const std::string& msg) + { + const auto end_time = GetTime<std::chrono::microseconds>() - m_start_t; + if (m_start_t.count() <= 0) { + return strprintf("%s: %s", m_prefix, msg); + } + + std::string units = ""; + float divisor = 1; + + if (std::is_same<TimeType, std::chrono::microseconds>::value) { + units = "μs"; + } else if (std::is_same<TimeType, std::chrono::milliseconds>::value) { + units = "ms"; + divisor = 1000.; + } else if (std::is_same<TimeType, std::chrono::seconds>::value) { + units = "s"; + divisor = 1000. * 1000.; + } + + const float time_ms = end_time.count() / divisor; + return strprintf("%s: %s (%.2f%s)", m_prefix, msg, time_ms, units); + } + +private: + std::chrono::microseconds m_start_t{}; + + //! Log prefix; usually the name of the function this was created in. + const std::string m_prefix{}; + + //! A descriptive message of what is being timed. + const std::string m_title{}; + + //! Forwarded on to LogPrint if specified - has the effect of only + //! outputing the timing log when a particular debug= category is specified. + const BCLog::LogFlags m_log_category{}; + +}; + +} // namespace BCLog + + +#define LOG_TIME_MICROS(end_msg, ...) \ + BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_MILLIS(end_msg, ...) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_SECONDS(end_msg, ...) \ + BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) + + +#endif // BITCOIN_LOGGING_TIMER_H @@ -808,7 +808,7 @@ public: bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; // Last time a "MEMPOOL" request was serviced. std::atomic<std::chrono::seconds> m_last_mempool_req{std::chrono::seconds{0}}; - int64_t nNextInvSend{0}; + std::chrono::microseconds nNextInvSend{0}; CCriticalSection cs_feeFilter; // Minimum fee rate with which to filter inv's to this node @@ -991,11 +991,13 @@ public: void MaybeSetAddrName(const std::string& addrNameIn); }; - - - - /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ int64_t PoissonNextSend(int64_t now, int average_interval_seconds); +/** Wrapper to return mockable type */ +inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, std::chrono::seconds average_interval) +{ + return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; +} + #endif // BITCOIN_NET_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d25a2d36e8..f42a26ca3e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3575,6 +3575,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Address refresh broadcast int64_t nNow = GetTimeMicros(); + auto current_time = GetTime<std::chrono::microseconds>(); + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { AdvertiseLocal(pto); pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); @@ -3796,13 +3798,13 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(pto->m_tx_relay->cs_tx_inventory); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(PF_NOBAN); - if (pto->m_tx_relay->nNextInvSend < nNow) { + if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; if (pto->fInbound) { - pto->m_tx_relay->nNextInvSend = connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL); + pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; } else { // Use half the delay for outbound peers, as there is less privacy concern for them. - pto->m_tx_relay->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> 1); + pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, std::chrono::seconds{INVENTORY_BROADCAST_INTERVAL >> 1}); } } @@ -3917,7 +3919,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling - const auto current_time = GetTime<std::chrono::microseconds>(); + current_time = GetTime<std::chrono::microseconds>(); // nNow is the current system time (GetTimeMicros is not mockable) and // should be replaced by the mockable current_time eventually nNow = GetTimeMicros(); diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 57fa158ad2..a818f06d51 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -14,9 +14,6 @@ #include <map> -#include <boost/thread.hpp> - - static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); @@ -38,6 +35,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); @@ -51,7 +49,6 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { @@ -61,6 +58,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c11aab8bd..a19af0fd1b 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,16 +15,17 @@ class CCoinsView; struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + CAmount nTotalAmount{0}; + + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 0000000000..702a0cbe53 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,50 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include <uint256.h> +#include <serialize.h> + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } + +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/noui.cpp b/src/noui.cpp index 11c8f1e13d..a5b7a2d591 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -66,28 +66,31 @@ void noui_connect() noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage); } -bool noui_ThreadSafeMessageBoxSuppressed(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -bool noui_ThreadSafeQuestionSuppressed(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -void noui_InitMessageSuppressed(const std::string& message) +void noui_InitMessageRedirect(const std::string& message) { + LogPrintf("init message: %s\n", message); } -void noui_suppress() +void noui_test_redirect() { noui_ThreadSafeMessageBoxConn.disconnect(); noui_ThreadSafeQuestionConn.disconnect(); noui_InitMessageConn.disconnect(); - noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxSuppressed); - noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionSuppressed); - noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageSuppressed); + noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxRedirect); + noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionRedirect); + noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageRedirect); } void noui_reconnect() diff --git a/src/noui.h b/src/noui.h index 854aeeacca..621e9c2798 100644 --- a/src/noui.h +++ b/src/noui.h @@ -17,10 +17,10 @@ void noui_InitMessage(const std::string& message); /** Connect all bitcoind signal handlers */ void noui_connect(); -/** Suppress all bitcoind signal handlers. Used to suppress output during test runs that produce expected errors */ -void noui_suppress(); +/** Redirect all bitcoind signal handlers to LogPrintf. Used to check or suppress output during test runs that produce expected errors */ +void noui_test_redirect(); -/** Reconnects the regular Non-GUI handlers after having used noui_suppress */ +/** Reconnects the regular Non-GUI handlers after having used noui_test_redirect */ void noui_reconnect(); #endif // BITCOIN_NOUI_H diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 386d559281..7190d59240 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -797,7 +797,7 @@ <item> <widget class="QPushButton" name="buttonMinimizeFee"> <property name="toolTip"> - <string>collapse fee-settings</string> + <string>Hide transaction fee settings</string> </property> <property name="text"> <string>Hide</string> diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index d08f852751..2f4b4412f5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -15,6 +15,7 @@ #include <hash.h> #include <index/blockfilterindex.h> #include <node/coinstats.h> +#include <node/utxo_snapshot.h> #include <policy/feerate.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -38,8 +39,6 @@ #include <univalue.h> -#include <boost/thread/thread.hpp> // boost::thread::interrupt - #include <condition_variable> #include <memory> #include <mutex> @@ -1975,7 +1974,6 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - boost::this_thread::interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -2245,6 +2243,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + "{\n" + " \"coins_written\": n, (numeric) the number of coins written in the snapshot\n" + " \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n" + " \"base_height\": n, (string) the height of the base of the snapshot\n" + " \"path\": \"...\" (string) the absolute path that the snapshot was written to\n" + "]\n" + }, + RPCExamples{ + HelpExampleCli("dumptxoutset", "utxo.dat") + } + }.Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE* file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr<CCoinsViewCursor> pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats + // based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2281,6 +2386,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index 0ba492c24e..a1dabee579 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -4,24 +4,12 @@ #include <bech32.h> #include <test/setup_common.h> +#include <test/util/str.h> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(bech32_tests, BasicTestingSetup) -static bool CaseInsensitiveEqual(const std::string &s1, const std::string &s2) -{ - if (s1.size() != s2.size()) return false; - for (size_t i = 0; i < s1.size(); ++i) { - char c1 = s1[i]; - if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); - char c2 = s2[i]; - if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); - if (c1 != c2) return false; - } - return true; -} - BOOST_AUTO_TEST_CASE(bip173_testvectors_valid) { static const std::string CASES[] = { diff --git a/src/test/fuzz/bech32.cpp b/src/test/fuzz/bech32.cpp new file mode 100644 index 0000000000..8b91f9bc96 --- /dev/null +++ b/src/test/fuzz/bech32.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bech32.h> +#include <test/fuzz/fuzz.h> +#include <test/util/str.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <utility> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_string(buffer.begin(), buffer.end()); + const std::pair<std::string, std::vector<uint8_t>> r1 = bech32::Decode(random_string); + if (r1.first.empty()) { + assert(r1.second.empty()); + } else { + const std::string& hrp = r1.first; + const std::vector<uint8_t>& data = r1.second; + const std::string reencoded = bech32::Encode(hrp, data); + assert(CaseInsensitiveEqual(random_string, reencoded)); + } + + std::vector<unsigned char> input; + ConvertBits<8, 5, true>([&](unsigned char c) { input.push_back(c); }, buffer.begin(), buffer.end()); + const std::string encoded = bech32::Encode("bc", input); + assert(!encoded.empty()); + + const std::pair<std::string, std::vector<uint8_t>> r2 = bech32::Decode(encoded); + if (r2.first.empty()) { + assert(r2.second.empty()); + } else { + const std::string& hrp = r2.first; + const std::vector<uint8_t>& data = r2.second; + assert(hrp == "bc"); + assert(data == input); + } +} diff --git a/src/test/lib/logging.cpp b/src/test/lib/logging.cpp new file mode 100644 index 0000000000..4cfebf63df --- /dev/null +++ b/src/test/lib/logging.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/lib/logging.h> + +#include <logging.h> +#include <noui.h> +#include <tinyformat.h> +#include <util/memory.h> + +#include <stdexcept> + +DebugLogHelper::DebugLogHelper(std::string message) + : m_message{std::move(message)} +{ + m_print_connection = LogInstance().PushBackCallback( + [this](const std::string& s) { + if (m_found) return; + m_found = s.find(m_message) != std::string::npos; + }); + noui_test_redirect(); +} + +void DebugLogHelper::check_found() +{ + noui_reconnect(); + LogInstance().DeleteCallback(m_print_connection); + if (!m_found) { + throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); + } +} diff --git a/src/test/lib/logging.h b/src/test/lib/logging.h new file mode 100644 index 0000000000..ea1b0ad6f0 --- /dev/null +++ b/src/test/lib/logging.h @@ -0,0 +1,29 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_LIB_LOGGING_H +#define BITCOIN_TEST_LIB_LOGGING_H + +#include <util/macros.h> + +#include <functional> +#include <list> +#include <string> + +class DebugLogHelper +{ + const std::string m_message; + bool m_found{false}; + std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + + void check_found(); + +public: + DebugLogHelper(std::string message); + ~DebugLogHelper() { check_found(); } +}; + +#define ASSERT_DEBUG_LOG(message) DebugLogHelper PASTE2(debugloghelper, __COUNTER__)(message) + +#endif // BITCOIN_TEST_LIB_LOGGING_H diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp new file mode 100644 index 0000000000..eb1826ae8d --- /dev/null +++ b/src/test/logging_tests.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <logging/timer.h> +#include <test/setup_common.h> + +#include <chrono> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(logging_timer) +{ + + SetMockTime(1); + auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); + + SetMockTime(1); + auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)"); + + SetMockTime(1); + auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000.00μs)"); + + SetMockTime(0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index fe5d31b7d3..6d2f5474ff 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -749,6 +749,43 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) pool.GetTransactionAncestry(ty6->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, 9ULL); BOOST_CHECK_EQUAL(descendants, 6ULL); + + /* Ancestors represented more than once ("diamond") */ + // + // [ta].0 <- [tb].0 -----<------- [td].0 + // | | + // \---1 <- [tc].0 --<--/ + // + CTransactionRef ta, tb, tc, td; + ta = make_tx(/* output_values */ {10 * COIN}); + tb = make_tx(/* output_values */ {5 * COIN, 3 * COIN}, /* inputs */ {ta}); + tc = make_tx(/* output_values */ {2 * COIN}, /* inputs */ {tb}, /* input_indices */ {1}); + td = make_tx(/* output_values */ {6 * COIN}, /* inputs */ {tb, tc}, /* input_indices */ {0, 0}); + pool.clear(); + pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tc)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(td)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =================== =========== + // ta 1 (ta 4 (ta,tb,tc,td) + // tb 2 (ta,tb) 4 (ta,tb,tc,td) + // tc 3 (ta,tb,tc) 4 (ta,tb,tc,td) + // td 4 (ta,tb,tc,td) 4 (ta,tb,tc,td) + pool.GetTransactionAncestry(ta->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tb->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tc->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(td->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 4ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index f5f217b841..0c1452822d 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -301,5 +301,19 @@ BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) BOOST_CHECK_EQUAL(IsLocal(addr), false); } +BOOST_AUTO_TEST_CASE(PoissonNextSend) +{ + g_mock_deterministic_tests = true; + + int64_t now = 5000; + int average_interval_seconds = 600; + + auto poisson = ::PoissonNextSend(now, average_interval_seconds); + std::chrono::microseconds poisson_chrono = ::PoissonNextSend(std::chrono::microseconds{now}, std::chrono::seconds{average_interval_seconds}); + + BOOST_CHECK_EQUAL(poisson, poisson_chrono.count()); + + g_mock_deterministic_tests = false; +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 7b00222ab7..c1a6df72d5 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -5,6 +5,7 @@ #include <netaddress.h> #include <noui.h> +#include <test/lib/logging.h> #include <test/setup_common.h> #include <timedata.h> #include <warnings.h> @@ -59,9 +60,10 @@ BOOST_AUTO_TEST_CASE(addtimedata) MultiAddTimeData(3, DEFAULT_MAX_TIME_ADJUSTMENT + 1); // Filter size is 1 + 3 = 4: It is always initialized with a single element (offset 0) - noui_suppress(); - MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 - noui_reconnect(); + { + ASSERT_DEBUG_LOG("Please check that your computer's date and time are correct!"); + MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 + } BOOST_CHECK(GetWarnings("gui").find("clock is wrong") != std::string::npos); diff --git a/src/test/util/str.cpp b/src/test/util/str.cpp new file mode 100644 index 0000000000..c517fe44d9 --- /dev/null +++ b/src/test/util/str.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/str.h> + +#include <cstdint> +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2) +{ + if (s1.size() != s2.size()) return false; + for (size_t i = 0; i < s1.size(); ++i) { + char c1 = s1[i]; + if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); + char c2 = s2[i]; + if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); + if (c1 != c2) return false; + } + return true; +} diff --git a/src/test/util/str.h b/src/test/util/str.h new file mode 100644 index 0000000000..63629501e8 --- /dev/null +++ b/src/test/util/str.h @@ -0,0 +1,12 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_STR_H +#define BITCOIN_TEST_UTIL_STR_H + +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2); + +#endif // BITCOIN_TEST_UTIL_STR_H diff --git a/src/validation.cpp b/src/validation.cpp index 11072b6038..01803223d1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -18,6 +18,8 @@ #include <flatfile.h> #include <hash.h> #include <index/txindex.h> +#include <logging.h> +#include <logging/timer.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -2199,6 +2201,10 @@ bool CChainState::FlushStateToDisk( static int64_t nLastFlush = 0; std::set<int> setFilesToPrune; bool full_flush_completed = false; + + const size_t coins_count = CoinsTip().GetCacheSize(); + const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage(); + try { { bool fFlushForPrune = false; @@ -2206,8 +2212,12 @@ bool CChainState::FlushStateToDisk( LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { + LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH); + FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); } else { + LOG_TIME_MILLIS("find files to prune", BCLog::BENCH); + FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } @@ -2246,10 +2256,17 @@ bool CChainState::FlushStateToDisk( if (!CheckDiskSpace(GetBlocksDir())) { return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } - // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); + { + LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH); + + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + } + // Then update all block file information (which may refer to block and undo files). { + LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH); + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; vFiles.reserve(setDirtyFileInfo.size()); for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { @@ -2267,12 +2284,18 @@ bool CChainState::FlushStateToDisk( } } // Finally remove any pruned files - if (fFlushForPrune) + if (fFlushForPrune) { + LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH); + UnlinkPrunedFiles(setFilesToPrune); + } nLastWrite = nNow; } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { + LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000)); + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already diff --git a/src/validation.h b/src/validation.h index 7f9582adfd..e9127040bf 100644 --- a/src/validation.h +++ b/src/validation.h @@ -21,6 +21,7 @@ #include <txmempool.h> // For CTxMemPool::cs #include <txdb.h> #include <versionbits.h> +#include <serialize.h> #include <algorithm> #include <atomic> diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 3ef2f883c3..7f998ab450 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1598,7 +1598,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; - if (depth == -1 || tx.GetDepthInMainChain(*locked_chain) < depth) { + if (depth == -1 || abs(tx.GetDepthInMainChain(*locked_chain)) < depth) { ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 279542ffad..7a6907bc93 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -5,6 +5,7 @@ #include <boost/test/unit_test.hpp> #include <noui.h> +#include <test/lib/logging.h> #include <test/setup_common.h> #include <util/system.h> #include <wallet/test/init_test_fixture.h> @@ -34,28 +35,31 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) { SetWalletDir(m_walletdir_path_cases["nonexistent"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("does not exist"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_directory) { SetWalletDir(m_walletdir_path_cases["file"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is not a directory"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative) { SetWalletDir(m_walletdir_path_cases["relative"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is a relative path"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b10a5deedc..0b7dc256ad 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1850,32 +1850,37 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { + std::set<uint256> s; + return IsTrusted(locked_chain, s); +} + +bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const +{ // Quick answer in most cases - if (!locked_chain.checkFinalTx(*tx)) { - return false; - } + if (!locked_chain.checkFinalTx(*tx)) return false; int nDepth = GetDepthInMainChain(locked_chain); - if (nDepth >= 1) - return true; - if (nDepth < 0) - return false; - if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) // using wtx's cached debit - return false; + if (nDepth >= 1) return true; + if (nDepth < 0) return false; + // using wtx's cached debit + if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) return false; // Don't trust unconfirmed transactions from us unless they are in the mempool. - if (!InMempool()) - return false; + if (!InMempool()) return false; // Trusted if all inputs are from us and are in the mempool: for (const CTxIn& txin : tx->vin) { // Transactions not sent by us: not trusted const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); - if (parent == nullptr) - return false; + if (parent == nullptr) return false; const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; - if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) - return false; + // Check that this specific input being spent is trusted + if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; + // If we've already trusted this parent, continue + if (trusted_parents.count(parent->GetHash())) continue; + // Recurse to check that the parent is also trusted + if (!parent->IsTrusted(locked_chain, trusted_parents)) return false; + trusted_parents.insert(parent->GetHash()); } return true; } @@ -1961,10 +1966,11 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons { auto locked_chain = chain().lock(); LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(*locked_chain)}; + const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; @@ -2011,6 +2017,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const int min_depth = {coinControl ? coinControl->m_min_depth : DEFAULT_MIN_DEPTH}; const int max_depth = {coinControl ? coinControl->m_max_depth : DEFAULT_MAX_DEPTH}; + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; @@ -2032,7 +2039,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain, trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -3094,11 +3101,12 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: { LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& walletEntry : mapWallet) { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain, trusted_parents)) continue; if (wtx.IsImmatureCoinBase(locked_chain)) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 7d0fae0bc7..6c9b3f40ab 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -476,6 +476,7 @@ public: bool InMempool() const; bool IsTrusted(interfaces::Chain::Lock& locked_chain) const; + bool IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 index 0000000000..7527bdfb08 --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / 'regtest' / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index aefcdc5ec8..f6ea9ef682 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -31,7 +31,7 @@ importing the `TestShell` class from the `test_shell` sub-package. ``` >>> import sys >>> sys.path.insert(0, "/path/to/bitcoin/test/functional") ->>> from test_framework.test_shell import `TestShell` +>>> from test_framework.test_shell import TestShell ``` The following `TestShell` methods manage the lifetime of the underlying bitcoind @@ -51,8 +51,7 @@ The following sections demonstrate how to initialize, run, and shut down a ## 3. Initializing a `TestShell` object ``` ->>> test = TestShell() ->>> test.setup(num_nodes=2, setup_clean_chain=True) +>>> test = TestShell().setup(num_nodes=2, setup_clean_chain=True) 20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Initializing test directory /path/to/bitcoin_func_test_XXXXXXX ``` The `TestShell` forwards all functional test parameters of the parent @@ -66,8 +65,7 @@ temporary folder. If you need more bitcoind nodes than set by default (1), simply increase the `num_nodes` parameter during setup. ``` ->>> test2 = TestShell() ->>> test2.setup() +>>> test2 = TestShell().setup() TestShell is already running! ``` diff --git a/test/functional/test_framework/test_shell.py b/test/functional/test_framework/test_shell.py index 79da35b364..26df128f1f 100644 --- a/test/functional/test_framework/test_shell.py +++ b/test/functional/test_framework/test_shell.py @@ -29,8 +29,7 @@ class TestShell: # Num_nodes parameter must be set # by BitcoinTestFramework child class. - self.num_nodes = kwargs.get('num_nodes', 1) - kwargs.pop('num_nodes', None) + self.num_nodes = 1 # User parameters override default values. for key, value in kwargs.items(): @@ -43,6 +42,7 @@ class TestShell: super().setup() self.running = True + return self def shutdown(self): if not self.running: @@ -53,7 +53,7 @@ class TestShell: def reset(self): if self.running: - print("Shutdown TestWrapper before resetting!") + print("Shutdown TestShell before resetting!") else: self.num_nodes = None super().__init__() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6665d2e7f5..9b4a5d2030 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -191,6 +191,7 @@ BASE_SCRIPTS = [ 'rpc_uptime.py', 'wallet_resendwallettransactions.py', 'wallet_fallbackfee.py', + 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', 'wallet_create_tx.py', diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c50dcd987a..a5f9a047ed 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -109,13 +109,51 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + # Before `test_balance()`, we have had two nodes with a balance of 50 + # each and then we: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 60 from node B to node A with fee 0.01 + # + # Then we check the balances: + # + # 1) As is + # 2) With transaction 2 from above with 2x the fee + # + # Prior to #16766, in this situation, the node would immediately report + # a balance of 30 on node B as unconfirmed and trusted. + # + # After #16766, we show that balance as unconfirmed. + # + # The balance is indeed "trusted" and "confirmed" insofar as removing + # the mempool transactions would return at least that much money. But + # the algorithm after #16766 marks it as unconfirmed because the 'taint' + # tracking of transaction trust for summing balances doesn't consider + # which inputs belong to a user. In this case, the change output in + # question could be "destroyed" by replace the 1st transaction above. + # + # The post #16766 behavior is correct; we shouldn't be treating those + # funds as confirmed. If you want to rely on that specific UTXO existing + # which has given you that balance, you cannot, as a third party + # spending the other input would destroy that unconfirmed. + # + # For example, if the test transactions were: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 10 from node B to node A with fee 0.01 + # + # Then our node would report a confirmed balance of 40 + 50 - 10 = 80 + # BTC, which is more than would be available if transaction 1 were + # replaced. + + def test_balances(*, fee_node_1=0): # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1) # change from node 1's send + assert_equal(self.nodes[1].getbalance(), Decimal('0')) # node 1's send had an unsafe input # Same with minconf=0 assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) - assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('0')) # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago # TODO: fix getbalance tracking of coin spentness depth assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) @@ -125,9 +163,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('60')) assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('0')) - assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('30') - fee_node_1) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('30') - fee_node_1) test_balances(fee_node_1=Decimal('0.01')) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 4aeb393255..455e89e310 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -5,6 +5,7 @@ """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import BIP125_SEQUENCE_NUMBER from test_framework.util import ( assert_array_result, assert_equal, @@ -12,6 +13,7 @@ from test_framework.util import ( connect_nodes, ) +from decimal import Decimal class ListSinceBlockTest(BitcoinTestFramework): def set_test_params(self): @@ -33,6 +35,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_reorg() self.test_double_spend() self.test_double_send() + self.double_spends_filtered() def test_no_blockhash(self): txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) @@ -291,5 +294,51 @@ class ListSinceBlockTest(BitcoinTestFramework): if tx['txid'] == txid1: assert_equal(tx['confirmations'], 2) + def double_spends_filtered(self): + ''' + `listsinceblock` was returning conflicted transactions even if they + occurred before the specified cutoff blockhash + ''' + spending_node = self.nodes[2] + dest_address = spending_node.getnewaddress() + + tx_input = dict( + sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in spending_node.listunspent())) + rawtx = spending_node.createrawtransaction( + [tx_input], {dest_address: tx_input["amount"] - Decimal("0.00051000"), + spending_node.getrawchangeaddress(): Decimal("0.00050000")}) + signedtx = spending_node.signrawtransactionwithwallet(rawtx) + orig_tx_id = spending_node.sendrawtransaction(signedtx["hex"]) + original_tx = spending_node.gettransaction(orig_tx_id) + + double_tx = spending_node.bumpfee(orig_tx_id) + + # check that both transactions exist + block_hash = spending_node.listsinceblock( + spending_node.getblockhash(spending_node.getblockcount())) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, True) + assert_equal(double_found, True) + + lastblockhash = spending_node.generate(1)[0] + + # check that neither transaction exists + block_hash = spending_node.listsinceblock(lastblockhash) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, False) + assert_equal(double_found, False) + if __name__ == '__main__': ListSinceBlockTest().main() |