aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xcontrib/devtools/utxo_snapshot.sh44
-rw-r--r--doc/developer-notes.md2
-rw-r--r--doc/release-notes-15954.md4
-rw-r--r--src/Makefile.am2
-rw-r--r--src/Makefile.test.include3
-rw-r--r--src/init.cpp16
-rw-r--r--src/logging.cpp7
-rw-r--r--src/logging.h20
-rw-r--r--src/logging/timer.h104
-rw-r--r--src/node/coinstats.cpp2
-rw-r--r--src/node/coinstats.h21
-rw-r--r--src/node/utxo_snapshot.h50
-rw-r--r--src/noui.cpp17
-rw-r--r--src/noui.h6
-rw-r--r--src/qt/forms/sendcoinsdialog.ui2
-rw-r--r--src/rpc/blockchain.cpp109
-rw-r--r--src/test/lib/logging.cpp32
-rw-r--r--src/test/lib/logging.h29
-rw-r--r--src/test/logging_tests.cpp36
-rw-r--r--src/test/timedata_tests.cpp8
-rw-r--r--src/validation.cpp29
-rw-r--r--src/validation.h1
-rw-r--r--src/wallet/test/init_tests.cpp28
-rwxr-xr-xtest/functional/rpc_dumptxoutset.py51
-rwxr-xr-xtest/functional/test_runner.py1
25 files changed, 568 insertions, 56 deletions
diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh
new file mode 100755
index 0000000000..dee25ff67b
--- /dev/null
+++ b/contrib/devtools/utxo_snapshot.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+export LC_ALL=C
+
+set -ueo pipefail
+
+if (( $# < 3 )); then
+ echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>'
+ echo
+ echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the "
+ echo " expected assumeutxo hash"
+ echo
+ echo 'Examples:'
+ echo
+ echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata"
+ echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli'
+ exit 1
+fi
+
+GENERATE_AT_HEIGHT="${1}"; shift;
+OUTPUT_PATH="${1}"; shift;
+# Most of the calls we make take a while to run, so pad with a lengthy timeout.
+BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999"
+
+# Block we'll invalidate/reconsider to rewind/fast-forward the chain.
+PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) )
+
+(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while")
+${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
+
+if [[ "${OUTPUT_PATH}" = "-" ]]; then
+ (>&2 echo "Generating txoutset info...")
+ ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g'
+else
+ (>&2 echo "Generating UTXO snapshot...")
+ ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"
+fi
+
+(>&2 echo "Restoring chain to original height; this may take a while")
+${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}"
diff --git a/doc/developer-notes.md b/doc/developer-notes.md
index 1756648157..e7fd8102a4 100644
--- a/doc/developer-notes.md
+++ b/doc/developer-notes.md
@@ -384,7 +384,7 @@ Threads
- ThreadScriptCheck : Verifies block scripts.
-- ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat.
+- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`.
- StartNode : Starts other threads.
diff --git a/doc/release-notes-15954.md b/doc/release-notes-15954.md
new file mode 100644
index 0000000000..f4d2c5688c
--- /dev/null
+++ b/doc/release-notes-15954.md
@@ -0,0 +1,4 @@
+Configuration option changes
+-----------------------------
+
+Importing blocks upon startup via the `bootstrap.dat` file no longer occurs by default. The file must now be specified with `-loadblock=<file>`.
diff --git a/src/Makefile.am b/src/Makefile.am
index 507e5cbb9f..ff4f071a3c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -146,6 +146,7 @@ BITCOIN_CORE_H = \
dbwrapper.h \
limitedmap.h \
logging.h \
+ logging/timer.h \
memusage.h \
merkleblock.h \
miner.h \
@@ -161,6 +162,7 @@ BITCOIN_CORE_H = \
node/context.h \
node/psbt.h \
node/transaction.h \
+ node/utxo_snapshot.h \
noui.h \
optional.h \
outputtype.h \
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index 48afa815d6..742530c55c 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -59,6 +59,8 @@ GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.r
BITCOIN_TEST_SUITE = \
test/lib/blockfilter.cpp \
test/lib/blockfilter.h \
+ test/lib/logging.cpp \
+ test/lib/logging.h \
test/lib/transaction_utils.cpp \
test/lib/transaction_utils.h \
test/main.cpp \
@@ -125,6 +127,7 @@ BITCOIN_TESTS =\
test/key_io_tests.cpp \
test/key_tests.cpp \
test/limitedmap_tests.cpp \
+ test/logging_tests.cpp \
test/dbwrapper_tests.cpp \
test/validation_tests.cpp \
test/mempool_tests.cpp \
diff --git a/src/init.cpp b/src/init.cpp
index 1a99ca9abc..3eb8fc7976 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -372,7 +372,7 @@ void SetupServerArgs()
gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
- gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+ gArgs.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -684,20 +684,6 @@ static void ThreadImport(std::vector<fs::path> vImportFiles)
LoadGenesisBlock(chainparams);
}
- // hardcoded $DATADIR/bootstrap.dat
- fs::path pathBootstrap = GetDataDir() / "bootstrap.dat";
- if (fs::exists(pathBootstrap)) {
- FILE *file = fsbridge::fopen(pathBootstrap, "rb");
- if (file) {
- fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old";
- LogPrintf("Importing bootstrap.dat...\n");
- LoadExternalBlockFile(chainparams, file);
- RenameOver(pathBootstrap, pathBootstrapOld);
- } else {
- LogPrintf("Warning: Could not open bootstrap file %s\n", pathBootstrap.string());
- }
- }
-
// -loadblock=
for (const fs::path& path : vImportFiles) {
FILE *file = fsbridge::fopen(path, "rb");
diff --git a/src/logging.cpp b/src/logging.cpp
index 60ab486198..b01177f23f 100644
--- a/src/logging.cpp
+++ b/src/logging.cpp
@@ -67,6 +67,9 @@ bool BCLog::Logger::StartLogging()
if (m_print_to_file) FileWriteStr(s, m_fileout);
if (m_print_to_console) fwrite(s.data(), 1, s.size(), stdout);
+ for (const auto& cb : m_print_callbacks) {
+ cb(s);
+ }
m_msgs_before_open.pop_front();
}
@@ -81,6 +84,7 @@ void BCLog::Logger::DisconnectTestLogger()
m_buffering = true;
if (m_fileout != nullptr) fclose(m_fileout);
m_fileout = nullptr;
+ m_print_callbacks.clear();
}
void BCLog::Logger::EnableCategory(BCLog::LogFlags flag)
@@ -270,6 +274,9 @@ void BCLog::Logger::LogPrintStr(const std::string& str)
fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout);
fflush(stdout);
}
+ for (const auto& cb : m_print_callbacks) {
+ cb(str_prefixed);
+ }
if (m_print_to_file) {
assert(m_fileout != nullptr);
diff --git a/src/logging.h b/src/logging.h
index e37c0c823b..9ed41c2b98 100644
--- a/src/logging.h
+++ b/src/logging.h
@@ -77,6 +77,9 @@ namespace BCLog {
std::string LogTimestampStr(const std::string& str);
+ /** Slots that connect to the print signal */
+ std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {};
+
public:
bool m_print_to_console = false;
bool m_print_to_file = false;
@@ -95,7 +98,22 @@ namespace BCLog {
bool Enabled() const
{
std::lock_guard<std::mutex> scoped_lock(m_cs);
- return m_buffering || m_print_to_console || m_print_to_file;
+ return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty();
+ }
+
+ /** Connect a slot to the print signal and return the connection */
+ std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun)
+ {
+ std::lock_guard<std::mutex> scoped_lock(m_cs);
+ m_print_callbacks.push_back(std::move(fun));
+ return --m_print_callbacks.end();
+ }
+
+ /** Delete a connection */
+ void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it)
+ {
+ std::lock_guard<std::mutex> scoped_lock(m_cs);
+ m_print_callbacks.erase(it);
}
/** Start logging (and flush all buffered messages) */
diff --git a/src/logging/timer.h b/src/logging/timer.h
new file mode 100644
index 0000000000..34dbb942c5
--- /dev/null
+++ b/src/logging/timer.h
@@ -0,0 +1,104 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_LOGGING_TIMER_H
+#define BITCOIN_LOGGING_TIMER_H
+
+#include <logging.h>
+#include <util/macros.h>
+#include <util/time.h>
+
+#include <chrono>
+#include <string>
+
+
+namespace BCLog {
+
+//! RAII-style object that outputs timing information to logs.
+template <typename TimeType>
+class Timer
+{
+public:
+ //! If log_category is left as the default, end_msg will log unconditionally
+ //! (instead of being filtered by category).
+ Timer(
+ std::string prefix,
+ std::string end_msg,
+ BCLog::LogFlags log_category = BCLog::LogFlags::ALL) :
+ m_prefix(std::move(prefix)),
+ m_title(std::move(end_msg)),
+ m_log_category(log_category)
+ {
+ this->Log(strprintf("%s started", m_title));
+ m_start_t = GetTime<std::chrono::microseconds>();
+ }
+
+ ~Timer()
+ {
+ this->Log(strprintf("%s completed", m_title));
+ }
+
+ void Log(const std::string& msg)
+ {
+ const std::string full_msg = this->LogMsg(msg);
+
+ if (m_log_category == BCLog::LogFlags::ALL) {
+ LogPrintf("%s\n", full_msg);
+ } else {
+ LogPrint(m_log_category, "%s\n", full_msg);
+ }
+ }
+
+ std::string LogMsg(const std::string& msg)
+ {
+ const auto end_time = GetTime<std::chrono::microseconds>() - m_start_t;
+ if (m_start_t.count() <= 0) {
+ return strprintf("%s: %s", m_prefix, msg);
+ }
+
+ std::string units = "";
+ float divisor = 1;
+
+ if (std::is_same<TimeType, std::chrono::microseconds>::value) {
+ units = "μs";
+ } else if (std::is_same<TimeType, std::chrono::milliseconds>::value) {
+ units = "ms";
+ divisor = 1000.;
+ } else if (std::is_same<TimeType, std::chrono::seconds>::value) {
+ units = "s";
+ divisor = 1000. * 1000.;
+ }
+
+ const float time_ms = end_time.count() / divisor;
+ return strprintf("%s: %s (%.2f%s)", m_prefix, msg, time_ms, units);
+ }
+
+private:
+ std::chrono::microseconds m_start_t{};
+
+ //! Log prefix; usually the name of the function this was created in.
+ const std::string m_prefix{};
+
+ //! A descriptive message of what is being timed.
+ const std::string m_title{};
+
+ //! Forwarded on to LogPrint if specified - has the effect of only
+ //! outputing the timing log when a particular debug= category is specified.
+ const BCLog::LogFlags m_log_category{};
+
+};
+
+} // namespace BCLog
+
+
+#define LOG_TIME_MICROS(end_msg, ...) \
+ BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__)
+#define LOG_TIME_MILLIS(end_msg, ...) \
+ BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__)
+#define LOG_TIME_SECONDS(end_msg, ...) \
+ BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__)
+
+
+#endif // BITCOIN_LOGGING_TIMER_H
diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp
index 049868736c..a818f06d51 100644
--- a/src/node/coinstats.cpp
+++ b/src/node/coinstats.cpp
@@ -35,6 +35,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash,
//! Calculate statistics about the unspent transaction output set
bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
{
+ stats = CCoinsStats();
std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
assert(pcursor);
@@ -57,6 +58,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
}
prevkey = key.hash;
outputs[key.n] = std::move(coin);
+ stats.coins_count++;
} else {
return error("%s: unable to read value", __func__);
}
diff --git a/src/node/coinstats.h b/src/node/coinstats.h
index 7c11aab8bd..a19af0fd1b 100644
--- a/src/node/coinstats.h
+++ b/src/node/coinstats.h
@@ -15,16 +15,17 @@ class CCoinsView;
struct CCoinsStats
{
- int nHeight;
- uint256 hashBlock;
- uint64_t nTransactions;
- uint64_t nTransactionOutputs;
- uint64_t nBogoSize;
- uint256 hashSerialized;
- uint64_t nDiskSize;
- CAmount nTotalAmount;
-
- CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {}
+ int nHeight{0};
+ uint256 hashBlock{};
+ uint64_t nTransactions{0};
+ uint64_t nTransactionOutputs{0};
+ uint64_t nBogoSize{0};
+ uint256 hashSerialized{};
+ uint64_t nDiskSize{0};
+ CAmount nTotalAmount{0};
+
+ //! The number of coins contained.
+ uint64_t coins_count{0};
};
//! Calculate statistics about the unspent transaction output set
diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h
new file mode 100644
index 0000000000..702a0cbe53
--- /dev/null
+++ b/src/node/utxo_snapshot.h
@@ -0,0 +1,50 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H
+#define BITCOIN_NODE_UTXO_SNAPSHOT_H
+
+#include <uint256.h>
+#include <serialize.h>
+
+//! Metadata describing a serialized version of a UTXO set from which an
+//! assumeutxo CChainState can be constructed.
+class SnapshotMetadata
+{
+public:
+ //! The hash of the block that reflects the tip of the chain for the
+ //! UTXO set contained in this snapshot.
+ uint256 m_base_blockhash;
+
+ //! The number of coins in the UTXO set contained in this snapshot. Used
+ //! during snapshot load to estimate progress of UTXO set reconstruction.
+ uint64_t m_coins_count = 0;
+
+ //! Necessary to "fake" the base nChainTx so that we can estimate progress during
+ //! initial block download for the assumeutxo chainstate.
+ unsigned int m_nchaintx = 0;
+
+ SnapshotMetadata() { }
+ SnapshotMetadata(
+ const uint256& base_blockhash,
+ uint64_t coins_count,
+ unsigned int nchaintx) :
+ m_base_blockhash(base_blockhash),
+ m_coins_count(coins_count),
+ m_nchaintx(nchaintx) { }
+
+ ADD_SERIALIZE_METHODS;
+
+ template <typename Stream, typename Operation>
+ inline void SerializationOp(Stream& s, Operation ser_action)
+ {
+ READWRITE(m_base_blockhash);
+ READWRITE(m_coins_count);
+ READWRITE(m_nchaintx);
+ }
+
+};
+
+#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H
diff --git a/src/noui.cpp b/src/noui.cpp
index 11c8f1e13d..a5b7a2d591 100644
--- a/src/noui.cpp
+++ b/src/noui.cpp
@@ -66,28 +66,31 @@ void noui_connect()
noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage);
}
-bool noui_ThreadSafeMessageBoxSuppressed(const std::string& message, const std::string& caption, unsigned int style)
+bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style)
{
+ LogPrintf("%s: %s\n", caption, message);
return false;
}
-bool noui_ThreadSafeQuestionSuppressed(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style)
+bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style)
{
+ LogPrintf("%s: %s\n", caption, message);
return false;
}
-void noui_InitMessageSuppressed(const std::string& message)
+void noui_InitMessageRedirect(const std::string& message)
{
+ LogPrintf("init message: %s\n", message);
}
-void noui_suppress()
+void noui_test_redirect()
{
noui_ThreadSafeMessageBoxConn.disconnect();
noui_ThreadSafeQuestionConn.disconnect();
noui_InitMessageConn.disconnect();
- noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxSuppressed);
- noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionSuppressed);
- noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageSuppressed);
+ noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxRedirect);
+ noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionRedirect);
+ noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageRedirect);
}
void noui_reconnect()
diff --git a/src/noui.h b/src/noui.h
index 854aeeacca..621e9c2798 100644
--- a/src/noui.h
+++ b/src/noui.h
@@ -17,10 +17,10 @@ void noui_InitMessage(const std::string& message);
/** Connect all bitcoind signal handlers */
void noui_connect();
-/** Suppress all bitcoind signal handlers. Used to suppress output during test runs that produce expected errors */
-void noui_suppress();
+/** Redirect all bitcoind signal handlers to LogPrintf. Used to check or suppress output during test runs that produce expected errors */
+void noui_test_redirect();
-/** Reconnects the regular Non-GUI handlers after having used noui_suppress */
+/** Reconnects the regular Non-GUI handlers after having used noui_test_redirect */
void noui_reconnect();
#endif // BITCOIN_NOUI_H
diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui
index 386d559281..7190d59240 100644
--- a/src/qt/forms/sendcoinsdialog.ui
+++ b/src/qt/forms/sendcoinsdialog.ui
@@ -797,7 +797,7 @@
<item>
<widget class="QPushButton" name="buttonMinimizeFee">
<property name="toolTip">
- <string>collapse fee-settings</string>
+ <string>Hide transaction fee settings</string>
</property>
<property name="text">
<string>Hide</string>
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index faed7d1317..2f4b4412f5 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -15,6 +15,7 @@
#include <hash.h>
#include <index/blockfilterindex.h>
#include <node/coinstats.h>
+#include <node/utxo_snapshot.h>
#include <policy/feerate.h>
#include <policy/policy.h>
#include <policy/rbf.h>
@@ -2242,6 +2243,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request)
return ret;
}
+/**
+ * Serialize the UTXO set to a file for loading elsewhere.
+ *
+ * @see SnapshotMetadata
+ */
+UniValue dumptxoutset(const JSONRPCRequest& request)
+{
+ RPCHelpMan{
+ "dumptxoutset",
+ "\nWrite the serialized UTXO set to disk.\n"
+ "Incidentally flushes the latest coinsdb (leveldb) to disk.\n",
+ {
+ {"path",
+ RPCArg::Type::STR,
+ RPCArg::Optional::NO,
+ /* default_val */ "",
+ "path to the output file. If relative, will be prefixed by datadir."},
+ },
+ RPCResult{
+ "{\n"
+ " \"coins_written\": n, (numeric) the number of coins written in the snapshot\n"
+ " \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n"
+ " \"base_height\": n, (string) the height of the base of the snapshot\n"
+ " \"path\": \"...\" (string) the absolute path that the snapshot was written to\n"
+ "]\n"
+ },
+ RPCExamples{
+ HelpExampleCli("dumptxoutset", "utxo.dat")
+ }
+ }.Check(request);
+
+ fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir());
+ // Write to a temporary path and then move into `path` on completion
+ // to avoid confusion due to an interruption.
+ fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir());
+
+ if (fs::exists(path)) {
+ throw JSONRPCError(
+ RPC_INVALID_PARAMETER,
+ path.string() + " already exists. If you are sure this is what you want, "
+ "move it out of the way first");
+ }
+
+ FILE* file{fsbridge::fopen(temppath, "wb")};
+ CAutoFile afile{file, SER_DISK, CLIENT_VERSION};
+ std::unique_ptr<CCoinsViewCursor> pcursor;
+ CCoinsStats stats;
+ CBlockIndex* tip;
+
+ {
+ // We need to lock cs_main to ensure that the coinsdb isn't written to
+ // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats
+ // based upon the coinsdb, and (iii) constructing a cursor to the
+ // coinsdb for use below this block.
+ //
+ // Cursors returned by leveldb iterate over snapshots, so the contents
+ // of the pcursor will not be affected by simultaneous writes during
+ // use below this block.
+ //
+ // See discussion here:
+ // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369
+ //
+ LOCK(::cs_main);
+
+ ::ChainstateActive().ForceFlushStateToDisk();
+
+ if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) {
+ throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
+ }
+
+ pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor());
+ tip = LookupBlockIndex(stats.hashBlock);
+ CHECK_NONFATAL(tip);
+ }
+
+ SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx};
+
+ afile << metadata;
+
+ COutPoint key;
+ Coin coin;
+ unsigned int iter{0};
+
+ while (pcursor->Valid()) {
+ if (iter % 5000 == 0 && !IsRPCRunning()) {
+ throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down");
+ }
+ ++iter;
+ if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
+ afile << key;
+ afile << coin;
+ }
+
+ pcursor->Next();
+ }
+
+ afile.fclose();
+ fs::rename(temppath, path);
+
+ UniValue result(UniValue::VOBJ);
+ result.pushKV("coins_written", stats.coins_count);
+ result.pushKV("base_hash", tip->GetBlockHash().ToString());
+ result.pushKV("base_height", tip->nHeight);
+ result.pushKV("path", path.string());
+ return result;
+}
+
// clang-format off
static const CRPCCommand commands[] =
{ // category name actor (function) argNames
@@ -2278,6 +2386,7 @@ static const CRPCCommand commands[] =
{ "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} },
{ "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} },
{ "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} },
+ { "hidden", "dumptxoutset", &dumptxoutset, {"path"} },
};
// clang-format on
diff --git a/src/test/lib/logging.cpp b/src/test/lib/logging.cpp
new file mode 100644
index 0000000000..4cfebf63df
--- /dev/null
+++ b/src/test/lib/logging.cpp
@@ -0,0 +1,32 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <test/lib/logging.h>
+
+#include <logging.h>
+#include <noui.h>
+#include <tinyformat.h>
+#include <util/memory.h>
+
+#include <stdexcept>
+
+DebugLogHelper::DebugLogHelper(std::string message)
+ : m_message{std::move(message)}
+{
+ m_print_connection = LogInstance().PushBackCallback(
+ [this](const std::string& s) {
+ if (m_found) return;
+ m_found = s.find(m_message) != std::string::npos;
+ });
+ noui_test_redirect();
+}
+
+void DebugLogHelper::check_found()
+{
+ noui_reconnect();
+ LogInstance().DeleteCallback(m_print_connection);
+ if (!m_found) {
+ throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message));
+ }
+}
diff --git a/src/test/lib/logging.h b/src/test/lib/logging.h
new file mode 100644
index 0000000000..ea1b0ad6f0
--- /dev/null
+++ b/src/test/lib/logging.h
@@ -0,0 +1,29 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_TEST_LIB_LOGGING_H
+#define BITCOIN_TEST_LIB_LOGGING_H
+
+#include <util/macros.h>
+
+#include <functional>
+#include <list>
+#include <string>
+
+class DebugLogHelper
+{
+ const std::string m_message;
+ bool m_found{false};
+ std::list<std::function<void(const std::string&)>>::iterator m_print_connection;
+
+ void check_found();
+
+public:
+ DebugLogHelper(std::string message);
+ ~DebugLogHelper() { check_found(); }
+};
+
+#define ASSERT_DEBUG_LOG(message) DebugLogHelper PASTE2(debugloghelper, __COUNTER__)(message)
+
+#endif // BITCOIN_TEST_LIB_LOGGING_H
diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp
new file mode 100644
index 0000000000..eb1826ae8d
--- /dev/null
+++ b/src/test/logging_tests.cpp
@@ -0,0 +1,36 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <logging.h>
+#include <logging/timer.h>
+#include <test/setup_common.h>
+
+#include <chrono>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(logging_timer)
+{
+
+ SetMockTime(1);
+ auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg");
+ SetMockTime(2);
+ BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)");
+
+ SetMockTime(1);
+ auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg");
+ SetMockTime(2);
+ BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)");
+
+ SetMockTime(1);
+ auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg");
+ SetMockTime(2);
+ BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000.00μs)");
+
+ SetMockTime(0);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp
index 7b00222ab7..c1a6df72d5 100644
--- a/src/test/timedata_tests.cpp
+++ b/src/test/timedata_tests.cpp
@@ -5,6 +5,7 @@
#include <netaddress.h>
#include <noui.h>
+#include <test/lib/logging.h>
#include <test/setup_common.h>
#include <timedata.h>
#include <warnings.h>
@@ -59,9 +60,10 @@ BOOST_AUTO_TEST_CASE(addtimedata)
MultiAddTimeData(3, DEFAULT_MAX_TIME_ADJUSTMENT + 1);
// Filter size is 1 + 3 = 4: It is always initialized with a single element (offset 0)
- noui_suppress();
- MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5
- noui_reconnect();
+ {
+ ASSERT_DEBUG_LOG("Please check that your computer's date and time are correct!");
+ MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5
+ }
BOOST_CHECK(GetWarnings("gui").find("clock is wrong") != std::string::npos);
diff --git a/src/validation.cpp b/src/validation.cpp
index 11072b6038..01803223d1 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -18,6 +18,8 @@
#include <flatfile.h>
#include <hash.h>
#include <index/txindex.h>
+#include <logging.h>
+#include <logging/timer.h>
#include <policy/fees.h>
#include <policy/policy.h>
#include <policy/settings.h>
@@ -2199,6 +2201,10 @@ bool CChainState::FlushStateToDisk(
static int64_t nLastFlush = 0;
std::set<int> setFilesToPrune;
bool full_flush_completed = false;
+
+ const size_t coins_count = CoinsTip().GetCacheSize();
+ const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage();
+
try {
{
bool fFlushForPrune = false;
@@ -2206,8 +2212,12 @@ bool CChainState::FlushStateToDisk(
LOCK(cs_LastBlockFile);
if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) {
if (nManualPruneHeight > 0) {
+ LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH);
+
FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight);
} else {
+ LOG_TIME_MILLIS("find files to prune", BCLog::BENCH);
+
FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
fCheckForPruning = false;
}
@@ -2246,10 +2256,17 @@ bool CChainState::FlushStateToDisk(
if (!CheckDiskSpace(GetBlocksDir())) {
return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX);
}
- // First make sure all block and undo data is flushed to disk.
- FlushBlockFile();
+ {
+ LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH);
+
+ // First make sure all block and undo data is flushed to disk.
+ FlushBlockFile();
+ }
+
// Then update all block file information (which may refer to block and undo files).
{
+ LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH);
+
std::vector<std::pair<int, const CBlockFileInfo*> > vFiles;
vFiles.reserve(setDirtyFileInfo.size());
for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) {
@@ -2267,12 +2284,18 @@ bool CChainState::FlushStateToDisk(
}
}
// Finally remove any pruned files
- if (fFlushForPrune)
+ if (fFlushForPrune) {
+ LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH);
+
UnlinkPrunedFiles(setFilesToPrune);
+ }
nLastWrite = nNow;
}
// Flush best chain related state. This can only be done if the blocks / block index write was also done.
if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) {
+ LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)",
+ coins_count, coins_mem_usage / 1000));
+
// Typical Coin structures on disk are around 48 bytes in size.
// Pushing a new one to the database can cause it to be written
// twice (once in the log, and once in the tables). This is already
diff --git a/src/validation.h b/src/validation.h
index 7f9582adfd..e9127040bf 100644
--- a/src/validation.h
+++ b/src/validation.h
@@ -21,6 +21,7 @@
#include <txmempool.h> // For CTxMemPool::cs
#include <txdb.h>
#include <versionbits.h>
+#include <serialize.h>
#include <algorithm>
#include <atomic>
diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp
index 279542ffad..7a6907bc93 100644
--- a/src/wallet/test/init_tests.cpp
+++ b/src/wallet/test/init_tests.cpp
@@ -5,6 +5,7 @@
#include <boost/test/unit_test.hpp>
#include <noui.h>
+#include <test/lib/logging.h>
#include <test/setup_common.h>
#include <util/system.h>
#include <wallet/test/init_test_fixture.h>
@@ -34,28 +35,31 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom)
BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist)
{
SetWalletDir(m_walletdir_path_cases["nonexistent"]);
- noui_suppress();
- bool result = m_chain_client->verify();
- noui_reconnect();
- BOOST_CHECK(result == false);
+ {
+ ASSERT_DEBUG_LOG("does not exist");
+ bool result = m_chain_client->verify();
+ BOOST_CHECK(result == false);
+ }
}
BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_directory)
{
SetWalletDir(m_walletdir_path_cases["file"]);
- noui_suppress();
- bool result = m_chain_client->verify();
- noui_reconnect();
- BOOST_CHECK(result == false);
+ {
+ ASSERT_DEBUG_LOG("is not a directory");
+ bool result = m_chain_client->verify();
+ BOOST_CHECK(result == false);
+ }
}
BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative)
{
SetWalletDir(m_walletdir_path_cases["relative"]);
- noui_suppress();
- bool result = m_chain_client->verify();
- noui_reconnect();
- BOOST_CHECK(result == false);
+ {
+ ASSERT_DEBUG_LOG("is a relative path");
+ bool result = m_chain_client->verify();
+ BOOST_CHECK(result == false);
+ }
}
BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing)
diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py
new file mode 100755
index 0000000000..7527bdfb08
--- /dev/null
+++ b/test/functional/rpc_dumptxoutset.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the generation of UTXO snapshots using `dumptxoutset`.
+"""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, assert_raises_rpc_error
+
+import hashlib
+from pathlib import Path
+
+
+class DumptxoutsetTest(BitcoinTestFramework):
+ def set_test_params(self):
+ self.setup_clean_chain = True
+ self.num_nodes = 1
+
+ def run_test(self):
+ """Test a trivial usage of the dumptxoutset RPC command."""
+ node = self.nodes[0]
+ mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1
+ node.setmocktime(mocktime)
+ node.generate(100)
+
+ FILENAME = 'txoutset.dat'
+ out = node.dumptxoutset(FILENAME)
+ expected_path = Path(node.datadir) / 'regtest' / FILENAME
+
+ assert expected_path.is_file()
+
+ assert_equal(out['coins_written'], 100)
+ assert_equal(out['base_height'], 100)
+ assert_equal(out['path'], str(expected_path))
+ # Blockhash should be deterministic based on mocked time.
+ assert_equal(
+ out['base_hash'],
+ '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6')
+
+ with open(str(expected_path), 'rb') as f:
+ digest = hashlib.sha256(f.read()).hexdigest()
+ # UTXO snapshot hash should be deterministic based on mocked time.
+ assert_equal(
+ digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664')
+
+ # Specifying a path to an existing file will fail.
+ assert_raises_rpc_error(
+ -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME)
+
+if __name__ == '__main__':
+ DumptxoutsetTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 6665d2e7f5..9b4a5d2030 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -191,6 +191,7 @@ BASE_SCRIPTS = [
'rpc_uptime.py',
'wallet_resendwallettransactions.py',
'wallet_fallbackfee.py',
+ 'rpc_dumptxoutset.py',
'feature_minchainwork.py',
'rpc_getblockstats.py',
'wallet_create_tx.py',