diff options
74 files changed, 2003 insertions, 502 deletions
diff --git a/configure.ac b/configure.ac index 3addd31029..6177324001 100644 --- a/configure.ac +++ b/configure.ac @@ -244,12 +244,6 @@ AC_ARG_ENABLE([lcov-branch-coverage], [use_lcov_branch=yes], [use_lcov_branch=no]) -AC_ARG_ENABLE([threadlocal], - [AS_HELP_STRING([--enable-threadlocal], - [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enable if there is platform support)])], - [use_thread_local=$enableval], - [use_thread_local=auto]) - AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -1028,45 +1022,6 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ [AC_MSG_RESULT([no])] ) -if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then - TEMP_LDFLAGS="$LDFLAGS" - LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" - AC_MSG_CHECKING([for thread_local support]) - AC_LINK_IFELSE([AC_LANG_SOURCE([ - #include <thread> - static thread_local int foo = 0; - static void run_thread() { foo++;} - int main(){ - for(int i = 0; i < 10; i++) { std::thread(run_thread).detach();} - return foo; - } - ])], - [ - case $host in - *mingw*) - dnl mingw32's implementation of thread_local has also been shown to behave - dnl erroneously under concurrent usage; see: - dnl https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 - AC_MSG_RESULT([no]) - ;; - *freebsd*) - dnl FreeBSD's implementation of thread_local is also buggy (per - dnl https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) - AC_MSG_RESULT([no]) - ;; - *) - AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define if thread_local is supported.]) - AC_MSG_RESULT([yes]) - ;; - esac - ], - [ - AC_MSG_RESULT([no]) - ] - ) - LDFLAGS="$TEMP_LDFLAGS" -fi - dnl Check for different ways of gathering OS randomness AC_MSG_CHECKING([for Linux getrandom function]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index ec332d23eb..7640102172 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -74,6 +74,22 @@ major version via the `-deprecatedrpc=` command line option. The release notes of a new major release come with detailed instructions on what RPC features were deprecated and how to re-enable them temporarily. +## JSON-RPC 1.1 vs 2.0 + +The server recognizes [JSON-RPC v2.0](https://www.jsonrpc.org/specification) requests +and responds accordingly. A 2.0 request is identified by the presence of +`"jsonrpc": "2.0"` in the request body. If that key + value is not present in a request, +the legacy JSON-RPC v1.1 protocol is followed instead, which was the only available +protocol in previous releases. + +|| 1.1 | 2.0 | +|-|-|-| +| Request marker | `"version": "1.1"` (or none) | `"jsonrpc": "2.0"` | +| Response marker | (none) | `"jsonrpc": "2.0"` | +| `"error"` and `"result"` fields in response | both present | only one is present | +| HTTP codes in response | `200` unless there is any kind of RPC error (invalid parameters, method not found, etc) | Always `200` unless there is an actual HTTP server error (request parsing error, endpoint not found, etc) | +| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field | + ## Security The RPC interface allows other programs to control Bitcoin Core, diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index 0f05cdcba7..5f54fd6d9a 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -1,6 +1,6 @@ # NetBSD Build Guide -Updated for NetBSD [9.2](https://netbsd.org/releases/formal-9/NetBSD-9.2.html). +**Updated for NetBSD [10.0](https://netbsd.org/releases/formal-10/NetBSD-10.0.html)** This guide describes how to build bitcoind, command-line utilities, and GUI on NetBSD. @@ -12,23 +12,23 @@ Install the required dependencies the usual way you [install software on NetBSD] The example commands below use `pkgin`. ```bash -pkgin install autoconf automake libtool pkg-config git gmake boost libevent +pkgin install autoconf automake libtool pkg-config git gmake boost-headers libevent ``` NetBSD currently ships with an older version of `gcc` than is needed to build. You should upgrade your `gcc` and then pass this new version to the configure script. -For example, grab `gcc9`: +For example, grab `gcc12`: ``` -pkgin install gcc9 +pkgin install gcc12 ``` Then, when configuring, pass the following: ```bash ./configure ... - CC="/usr/pkg/gcc9/bin/gcc" \ - CXX="/usr/pkg/gcc9/bin/g++" \ + CC="/usr/pkg/gcc12/bin/gcc" \ + CXX="/usr/pkg/gcc12/bin/g++" \ ... ``` @@ -66,10 +66,10 @@ pkgin install db4 #### GUI Dependencies -Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install `qt5`. +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. ```bash -pkgin install qt5 +pkgin install qt5-qtbase qt5-qttools ``` The GUI can encode addresses in a QR Code. To build in QR support for the GUI, install `qrencode`. @@ -84,7 +84,7 @@ There is an included test suite that is useful for testing code changes when dev To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkgin install python37 +pkgin install python39 ``` ### Building Bitcoin Core diff --git a/doc/release-notes-27101.md b/doc/release-notes-27101.md new file mode 100644 index 0000000000..8775b59c00 --- /dev/null +++ b/doc/release-notes-27101.md @@ -0,0 +1,9 @@ +JSON-RPC +-------- + +The JSON-RPC server now recognizes JSON-RPC 2.0 requests and responds with +strict adherence to the specification (https://www.jsonrpc.org/specification): + +- Returning HTTP "204 No Content" responses to JSON-RPC 2.0 notifications instead of full responses. +- Returning HTTP "200 OK" responses in all other cases, rather than 404 responses for unknown methods, 500 responses for invalid parameters, etc. +- Returning either "result" fields or "error" fields in JSON-RPC responses, rather than returning both fields with one field set to null. diff --git a/src/Makefile.am b/src/Makefile.am index b749651b72..ad37928b4d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -348,6 +348,7 @@ BITCOIN_CORE_H = \ wallet/feebumper.h \ wallet/fees.h \ wallet/load.h \ + wallet/migrate.h \ wallet/receive.h \ wallet/rpc/util.h \ wallet/rpc/wallet.h \ @@ -508,6 +509,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/fees.cpp \ wallet/interfaces.cpp \ wallet/load.cpp \ + wallet/migrate.cpp \ wallet/receive.cpp \ wallet/rpc/addresses.cpp \ wallet/rpc/backup.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index cfd28b0a4d..62a82189c1 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -204,7 +204,8 @@ FUZZ_WALLET_SRC = \ wallet/test/fuzz/coincontrol.cpp \ wallet/test/fuzz/coinselection.cpp \ wallet/test/fuzz/fees.cpp \ - wallet/test/fuzz/parse_iso8601.cpp + wallet/test/fuzz/parse_iso8601.cpp \ + wallet/test/fuzz/wallet_bdb_parser.cpp if USE_SQLITE FUZZ_WALLET_SRC += \ diff --git a/src/bench/readblock.cpp b/src/bench/readblock.cpp index 0545c6b017..2b2bfe069e 100644 --- a/src/bench/readblock.cpp +++ b/src/bench/readblock.cpp @@ -18,7 +18,7 @@ static FlatFilePos WriteBlockToDisk(ChainstateManager& chainman) CBlock block; stream >> TX_WITH_WITNESS(block); - return chainman.m_blockman.SaveBlockToDisk(block, 0, nullptr); + return chainman.m_blockman.SaveBlockToDisk(block, 0); } static void ReadBlockFromDiskTest(benchmark::Bench& bench) diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 4927634233..4d2a6f0c2a 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -151,7 +151,7 @@ int main(int argc, char* argv[]) { LOCK(chainman.GetMutex()); std::cout - << "\t" << "Reindexing: " << std::boolalpha << node::fReindex.load() << std::noboolalpha << std::endl + << "\t" << "Reindexing: " << std::boolalpha << chainman.m_blockman.m_reindexing.load() << std::noboolalpha << std::endl << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl << "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index c7ba2204c3..b7e4e64103 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -298,7 +298,7 @@ public: } addresses.pushKV("total", total); result.pushKV("addresses_known", addresses); - return JSONRPCReplyObj(result, NullUniValue, 1); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); } }; @@ -367,7 +367,7 @@ public: } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); - return JSONRPCReplyObj(result, NullUniValue, 1); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); } }; @@ -622,7 +622,7 @@ public: } } - return JSONRPCReplyObj(UniValue{result}, NullUniValue, 1); + return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); } const std::string m_help_doc{ @@ -709,7 +709,7 @@ public: UniValue result(UniValue::VOBJ); result.pushKV("address", address_str); result.pushKV("blocks", reply.get_obj()["result"]); - return JSONRPCReplyObj(result, NullUniValue, 1); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); } protected: std::string address_str; diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index e6d20b55c2..87af347473 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -40,6 +40,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-withinternalbdb", "Use the internal Berkeley DB parser when dumping a Berkeley DB wallet file (default: false)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddCommand("info", "Get wallet info"); argsman.AddCommand("create", "Create new wallet file"); diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 9160ec19e6..42282c32d1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -53,6 +53,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-walletrejectlongchains", "-walletcrosschain", "-unsafesqlitesync", + "-swapbdbendian", }); } diff --git a/src/httprpc.cpp b/src/httprpc.cpp index c72dbf10bc..3eb34dbe6a 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -73,8 +73,11 @@ static std::vector<std::vector<std::string>> g_rpcauth; static std::map<std::string, std::set<std::string>> g_rpc_whitelist; static bool g_rpc_whitelist_default = false; -static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id) +static void JSONErrorReply(HTTPRequest* req, UniValue objError, const JSONRPCRequest& jreq) { + // Sending HTTP errors is a legacy JSON-RPC behavior. + Assume(jreq.m_json_version != JSONRPCVersion::V2); + // Send error reply from json-rpc error object int nStatus = HTTP_INTERNAL_SERVER_ERROR; int code = objError.find_value("code").getInt<int>(); @@ -84,7 +87,7 @@ static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const Uni else if (code == RPC_METHOD_NOT_FOUND) nStatus = HTTP_NOT_FOUND; - std::string strReply = JSONRPCReply(NullUniValue, objError, id); + std::string strReply = JSONRPCReplyObj(NullUniValue, std::move(objError), jreq.id, jreq.m_json_version).write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(nStatus, strReply); @@ -185,7 +188,7 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req) // Set the URI jreq.URI = req->GetURI(); - std::string strReply; + UniValue reply; bool user_has_whitelist = g_rpc_whitelist.count(jreq.authUser); if (!user_has_whitelist && g_rpc_whitelist_default) { LogPrintf("RPC User %s not allowed to call any methods\n", jreq.authUser); @@ -200,13 +203,23 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req) req->WriteReply(HTTP_FORBIDDEN); return false; } - UniValue result = tableRPC.execute(jreq); - // Send reply - strReply = JSONRPCReply(result, NullUniValue, jreq.id); + // Legacy 1.0/1.1 behavior is for failed requests to throw + // exceptions which return HTTP errors and RPC errors to the client. + // 2.0 behavior is to catch exceptions and return HTTP success with + // RPC errors, as long as there is not an actual HTTP server error. + const bool catch_errors{jreq.m_json_version == JSONRPCVersion::V2}; + reply = JSONRPCExec(jreq, catch_errors); + + if (jreq.IsNotification()) { + // Even though we do execute notifications, we do not respond to them + req->WriteReply(HTTP_NO_CONTENT); + return true; + } // array of requests } else if (valRequest.isArray()) { + // Check authorization for each request's method if (user_has_whitelist) { for (unsigned int reqIdx = 0; reqIdx < valRequest.size(); reqIdx++) { if (!valRequest[reqIdx].isObject()) { @@ -223,18 +236,49 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req) } } } - strReply = JSONRPCExecBatch(jreq, valRequest.get_array()); + + // Execute each request + reply = UniValue::VARR; + for (size_t i{0}; i < valRequest.size(); ++i) { + // Batches never throw HTTP errors, they are always just included + // in "HTTP OK" responses. Notifications never get any response. + UniValue response; + try { + jreq.parse(valRequest[i]); + response = JSONRPCExec(jreq, /*catch_errors=*/true); + } catch (UniValue& e) { + response = JSONRPCReplyObj(NullUniValue, std::move(e), jreq.id, jreq.m_json_version); + } catch (const std::exception& e) { + response = JSONRPCReplyObj(NullUniValue, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id, jreq.m_json_version); + } + if (!jreq.IsNotification()) { + reply.push_back(std::move(response)); + } + } + // Return no response for an all-notification batch, but only if the + // batch request is non-empty. Technically according to the JSON-RPC + // 2.0 spec, an empty batch request should also return no response, + // However, if the batch request is empty, it means the request did + // not contain any JSON-RPC version numbers, so returning an empty + // response could break backwards compatibility with old RPC clients + // relying on previous behavior. Return an empty array instead of an + // empty response in this case to favor being backwards compatible + // over complying with the JSON-RPC 2.0 spec in this case. + if (reply.size() == 0 && valRequest.size() > 0) { + req->WriteReply(HTTP_NO_CONTENT); + return true; + } } else throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); req->WriteHeader("Content-Type", "application/json"); - req->WriteReply(HTTP_OK, strReply); - } catch (const UniValue& objError) { - JSONErrorReply(req, objError, jreq.id); + req->WriteReply(HTTP_OK, reply.write() + "\n"); + } catch (UniValue& e) { + JSONErrorReply(req, std::move(e), jreq); return false; } catch (const std::exception& e) { - JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); + JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq); return false; } return true; diff --git a/src/init.cpp b/src/init.cpp index 51ade4de93..0aac2ac65f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -126,7 +126,6 @@ using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPATHEIGHT; -using node::fReindex; using node::KernelNotifications; using node::LoadChainstate; using node::MempoolPath; @@ -1483,7 +1482,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status); ReadNotificationArgs(args, *node.notifications); - fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, @@ -1560,7 +1558,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); - options.reindex = node::fReindex; + options.reindex = chainman.m_blockman.m_reindexing; options.reindex_chainstate = fReindexChainState; options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); @@ -1608,7 +1606,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { - fReindex = true; + chainman.m_blockman.m_reindexing = true; if (!Assert(node.shutdown)->reset()) { LogPrintf("Internal error: failed to reset shutdown signal.\n"); } @@ -1641,17 +1639,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); + g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, chainman.m_blockman.m_reindexing); node.indexes.emplace_back(g_txindex.get()); } for (const auto& filter_type : g_enabled_filter_types) { - InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, fReindex); + InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, chainman.m_blockman.m_reindexing); node.indexes.emplace_back(GetBlockFilterIndex(filter_type)); } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, fReindex); + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, chainman.m_blockman.m_reindexing); node.indexes.emplace_back(g_coin_stats_index.get()); } @@ -1670,7 +1668,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // if pruning, perform the initial blockstore prune // after any wallet rescanning has taken place. if (chainman.m_blockman.IsPruneMode()) { - if (!fReindex) { + if (!chainman.m_blockman.m_reindexing) { LOCK(cs_main); for (Chainstate* chainstate : chainman.GetAll()) { uiInterface.InitMessage(_("Pruning blockstore…").translated); @@ -1696,7 +1694,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) int chain_active_height = WITH_LOCK(cs_main, return chainman.ActiveChain().Height()); // On first startup, warn on low block storage space - if (!fReindex && !fReindexChainState && chain_active_height <= 1) { + if (!chainman.m_blockman.m_reindexing && !fReindexChainState && chain_active_height <= 1) { uint64_t assumed_chain_bytes{chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024}; uint64_t additional_bytes_needed{ chainman.m_blockman.IsPruneMode() ? diff --git a/src/init/common.cpp b/src/init/common.cpp index 3a6df3e8bd..f473ee6d66 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -31,11 +31,7 @@ void AddLoggingArgs(ArgsManager& argsman) argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-loglevel=<level>|<category>:<level>", strprintf("Set the global or per-category severity level for logging categories enabled with the -debug configuration option or the logging RPC. Possible values are %s (default=%s). The following levels are always logged: error, warning, info. If <category>:<level> is supplied, the setting will override the global one and may be specified multiple times to set multiple category-specific levels. <category> can be: %s.", LogInstance().LogLevelsString(), LogInstance().LogLevelToStr(BCLog::DEFAULT_LOG_LEVEL), LogInstance().LogCategoriesString()), ArgsManager::DISALLOW_NEGATION | ArgsManager::DISALLOW_ELISION | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); -#ifdef HAVE_THREAD_LOCAL - argsman.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); -#else - argsman.AddHiddenArgs({"-logthreadnames"}); -#endif + argsman.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logsourcelocations", strprintf("Prepend debug output with name of the originating source location (source file, line number and function name) (default: %u)", DEFAULT_LOGSOURCELOCATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-loglevelalways", strprintf("Always prepend a category and level (default: %u)", DEFAULT_LOGLEVELALWAYS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); @@ -50,9 +46,7 @@ void SetLoggingOptions(const ArgsManager& args) LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); -#ifdef HAVE_THREAD_LOCAL LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); -#endif LogInstance().m_log_sourcelocations = args.GetBoolArg("-logsourcelocations", DEFAULT_LOGSOURCELOCATIONS); LogInstance().m_always_print_category_level = args.GetBoolArg("-loglevelalways", DEFAULT_LOGLEVELALWAYS); diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index deeba7e318..16072b669b 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -24,6 +24,7 @@ struct BlockManagerOpts { bool fast_prune{false}; const fs::path blocks_dir; Notifications& notifications; + bool reindex{false}; }; } // namespace kernel diff --git a/src/net.cpp b/src/net.cpp index 472b8e517e..de974f39cb 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2831,7 +2831,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo(bool include_connected) co } for (const auto& addr : lAddresses) { - CService service(LookupNumeric(addr.m_added_node, GetDefaultPort(addr.m_added_node))); + CService service{MaybeFlipIPv6toCJDNS(LookupNumeric(addr.m_added_node, GetDefaultPort(addr.m_added_node)))}; AddedNodeInfo addedNode{addr, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port @@ -3736,8 +3736,9 @@ CNode::CNode(NodeId idIn, { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); - for (const std::string &msg : getAllNetMessageTypes()) + for (const auto& msg : ALL_NET_MESSAGE_TYPES) { mapRecvBytesPerMsgType[msg] = 0; + } mapRecvBytesPerMsgType[NET_MESSAGE_TYPE_OTHER] = 0; if (fLogIPs) { diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index fa76566652..dd8419a68a 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -33,6 +33,8 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Op if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; + if (auto value{args.GetBoolArg("-reindex")}) opts.reindex = *value; + return {}; } } // namespace node diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 76837f2a27..4067ccee51 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -150,7 +150,6 @@ bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, s } // namespace kernel namespace node { -std::atomic_bool fReindex(false); bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const { @@ -552,7 +551,7 @@ bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_block // Check whether we need to continue reindexing bool fReindexing = false; m_block_tree_db->ReadReindexing(fReindexing); - if (fReindexing) fReindex = true; + if (fReindexing) m_reindexing = true; return true; } @@ -848,7 +847,7 @@ fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const return BlockFileSeq().FileName(pos); } -bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown) +FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime) { LOCK(cs_LastBlockFile); @@ -863,88 +862,101 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne } const int last_blockfile = m_blockfile_cursors[chain_type]->file_num; - int nFile = fKnown ? pos.nFile : last_blockfile; + int nFile = last_blockfile; if (static_cast<int>(m_blockfile_info.size()) <= nFile) { m_blockfile_info.resize(nFile + 1); } bool finalize_undo = false; - if (!fKnown) { - unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; - // Use smaller blockfiles in test-only -fastprune mode - but avoid - // the possibility of having a block not fit into the block file. - if (m_opts.fast_prune) { - max_blockfile_size = 0x10000; // 64kiB - if (nAddSize >= max_blockfile_size) { - // dynamically adjust the blockfile size to be larger than the added size - max_blockfile_size = nAddSize + 1; - } + unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; + // Use smaller blockfiles in test-only -fastprune mode - but avoid + // the possibility of having a block not fit into the block file. + if (m_opts.fast_prune) { + max_blockfile_size = 0x10000; // 64kiB + if (nAddSize >= max_blockfile_size) { + // dynamically adjust the blockfile size to be larger than the added size + max_blockfile_size = nAddSize + 1; } - assert(nAddSize < max_blockfile_size); - - while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) { - // when the undo file is keeping up with the block file, we want to flush it explicitly - // when it is lagging behind (more blocks arrive than are being connected), we let the - // undo block write case handle it - finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) == - Assert(m_blockfile_cursors[chain_type])->undo_height); - - // Try the next unclaimed blockfile number - nFile = this->MaxBlockfileNum() + 1; - // Set to increment MaxBlockfileNum() for next iteration - m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; - - if (static_cast<int>(m_blockfile_info.size()) <= nFile) { - m_blockfile_info.resize(nFile + 1); - } + } + assert(nAddSize < max_blockfile_size); + + while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) == + Assert(m_blockfile_cursors[chain_type])->undo_height); + + // Try the next unclaimed blockfile number + nFile = this->MaxBlockfileNum() + 1; + // Set to increment MaxBlockfileNum() for next iteration + m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; + + if (static_cast<int>(m_blockfile_info.size()) <= nFile) { + m_blockfile_info.resize(nFile + 1); } - pos.nFile = nFile; - pos.nPos = m_blockfile_info[nFile].nSize; } + FlatFilePos pos; + pos.nFile = nFile; + pos.nPos = m_blockfile_info[nFile].nSize; if (nFile != last_blockfile) { - if (!fKnown) { - LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n", - last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight); - - // Do not propagate the return code. The flush concerns a previous block - // and undo file that has already been written to. If a flush fails - // here, and we crash, there is no expected additional block data - // inconsistency arising from the flush failure here. However, the undo - // data may be inconsistent after a crash if the flush is called during - // a reindex. A flush error might also leave some of the data files - // untrimmed. - if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) { - LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, - "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n", - last_blockfile, !fKnown, finalize_undo, nFile); - } + LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n", + last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight); + + // Do not propagate the return code. The flush concerns a previous block + // and undo file that has already been written to. If a flush fails + // here, and we crash, there is no expected additional block data + // inconsistency arising from the flush failure here. However, the undo + // data may be inconsistent after a crash if the flush is called during + // a reindex. A flush error might also leave some of the data files + // untrimmed. + if (!FlushBlockFile(last_blockfile, /*fFinalize=*/true, finalize_undo)) { + LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, + "Failed to flush previous block file %05i (finalize=1, finalize_undo=%i) before opening new block file %05i\n", + last_blockfile, finalize_undo, nFile); } // No undo data yet in the new file, so reset our undo-height tracking. m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; } m_blockfile_info[nFile].AddBlock(nHeight, nTime); - if (fKnown) { - m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize); - } else { - m_blockfile_info[nFile].nSize += nAddSize; + m_blockfile_info[nFile].nSize += nAddSize; + + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + m_opts.notifications.fatalError(_("Disk space is too low!")); + return {}; + } + if (bytes_allocated != 0 && IsPruneMode()) { + m_check_for_pruning = true; } - if (!fKnown) { - bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - m_opts.notifications.fatalError(_("Disk space is too low!")); - return false; - } - if (bytes_allocated != 0 && IsPruneMode()) { - m_check_for_pruning = true; - } + m_dirty_fileinfo.insert(nFile); + return pos; +} + +void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos) +{ + LOCK(cs_LastBlockFile); + + // Update the cursor so it points to the last file. + const BlockfileType chain_type{BlockfileTypeForHeight(nHeight)}; + auto& cursor{m_blockfile_cursors[chain_type]}; + if (!cursor || cursor->file_num < pos.nFile) { + m_blockfile_cursors[chain_type] = BlockfileCursor{pos.nFile}; } + // Update the file information with the current block. + const unsigned int added_size = ::GetSerializeSize(TX_WITH_WITNESS(block)); + const int nFile = pos.nFile; + if (static_cast<int>(m_blockfile_info.size()) <= nFile) { + m_blockfile_info.resize(nFile + 1); + } + m_blockfile_info[nFile].AddBlock(nHeight, block.GetBlockTime()); + m_blockfile_info[nFile].nSize = std::max(pos.nPos + added_size, m_blockfile_info[nFile].nSize); m_dirty_fileinfo.insert(nFile); - return true; } bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) @@ -1014,7 +1026,7 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height // in the block file info as below; note that this does not catch the case where the undo writes are keeping up // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in - // the FindBlockPos function + // the FindNextBlockPos function if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { // Do not propagate the return code, a failed flush here should not // be an indication for a failed write. If it were propagated here, @@ -1130,28 +1142,20 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF return true; } -FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight) { unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block)); - FlatFilePos blockPos; - const auto position_known {dbp != nullptr}; - if (position_known) { - blockPos = *dbp; - } else { - // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for - // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). - // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. - nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); - } - if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) { - LogError("%s: FindBlockPos failed\n", __func__); + // Account for the 4 magic message start bytes + the 4 length bytes (8 bytes total, + // defined as BLOCK_SERIALIZATION_HEADER_SIZE) + nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); + FlatFilePos blockPos{FindNextBlockPos(nBlockSize, nHeight, block.GetBlockTime())}; + if (blockPos.IsNull()) { + LogError("%s: FindNextBlockPos failed\n", __func__); return FlatFilePos(); } - if (!position_known) { - if (!WriteBlockToDisk(block, blockPos)) { - m_opts.notifications.fatalError(_("Failed to write block.")); - return FlatFilePos(); - } + if (!WriteBlockToDisk(block, blockPos)) { + m_opts.notifications.fatalError(_("Failed to write block.")); + return FlatFilePos(); } return blockPos; } @@ -1178,7 +1182,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile ImportingNow imp{chainman.m_blockman.m_importing}; // -reindex - if (fReindex) { + if (chainman.m_blockman.m_reindexing) { int nFile = 0; // Map of disk positions for blocks with unknown parent (only used for reindex); // parent hash -> child disk position, multiple children can have the same parent. @@ -1201,7 +1205,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile nFile++; } WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); - fReindex = false; + chainman.m_blockman.m_reindexing = false; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): chainman.ActiveChainstate().LoadGenesisBlock(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index ce514cc645..a501067091 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -76,8 +76,6 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Size of header written by WriteBlockToDisk before a serialized CBlock */ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int); -extern std::atomic_bool fReindex; - // Because validation code takes pointers to the map's CBlockIndex objects, if // we ever switch to another associative container, we need to either use a // container that has stable addressing (true of all std associative @@ -155,7 +153,16 @@ private: /** Return false if undo file flushing fails. */ [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false); - [[nodiscard]] bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown); + /** + * Helper function performing various preparations before a block can be saved to disk: + * Returns the correct position for the block to be saved, which may be in the current or a new + * block file depending on nAddSize. May flush the previous blockfile to disk if full, updates + * blockfile info, and checks if there is enough disk space to save the block. + * + * The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of + * separator fields which are written before it by WriteBlockToDisk (BLOCK_SERIALIZATION_HEADER_SIZE). + */ + [[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime); [[nodiscard]] bool FlushChainstateBlockFile(int tip_height); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); @@ -164,6 +171,12 @@ private: AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + /** + * Write a block to disk. The pos argument passed to this function is modified by this call. Before this call, it should + * point to an unused file location where separator fields will be written, followed by the serialized CBlock data. + * After this call, it will point to the beginning of the serialized CBlock data, after the separator fields + * (BLOCK_SERIALIZATION_HEADER_SIZE) + */ bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const; bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; @@ -206,7 +219,7 @@ private: //! effectively. //! //! This data structure maintains separate blockfile number cursors for each - //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindBlockPos(). + //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindNextBlockPos(). //! //! The first element is the NORMAL cursor, second is ASSUMED. std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES> @@ -254,11 +267,19 @@ public: explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)}, - m_interrupt{interrupt} {}; + m_interrupt{interrupt}, + m_reindexing{m_opts.reindex} {}; const util::SignalInterrupt& m_interrupt; std::atomic<bool> m_importing{false}; + /** + * Tracks if a reindex is currently in progress. Set to true when a reindex + * is requested and false when reindexing completes. Its value is persisted + * in the BlockTreeDB across restarts. + */ + std::atomic_bool m_reindexing; + BlockMap m_block_index GUARDED_BY(cs_main); /** @@ -312,8 +333,24 @@ public: bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ - FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp); + /** Store block on disk and update block file statistics. + * + * @param[in] block the block to be stored + * @param[in] nHeight the height of the block + * + * @returns in case of success, the position to which the block was written to + * in case of an error, an empty FlatFilePos + */ + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight); + + /** Update blockfile info while processing a block during reindex. The block must be available on disk. + * + * @param[in] block the block being processed + * @param[in] nHeight the height of the block + * @param[in] pos the position of the serialized CBlock on disk. This is the position returned + * by WriteBlockToDisk pointing at the CBlock, not the separator fields before it + */ + void UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } @@ -322,7 +359,7 @@ public: [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; } static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()}; - [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; } + [[nodiscard]] bool LoadingBlocks() const { return m_importing || m_reindexing; } /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index bf1fc06b0b..d6eb14f513 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -60,8 +60,8 @@ static ChainstateLoadResult CompleteChainstateInitialization( // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets fReindex global based on the disk flag! - // From here on, fReindex and options.reindex values may be different! + // Note that it also sets m_reindexing based on the disk flag! + // From here on, m_reindexing and options.reindex values may be different! if (!chainman.LoadBlockIndex()) { if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; @@ -84,7 +84,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). // This is called again in ImportBlocks after the reindex completes. - if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock()) { + if (!chainman.m_blockman.m_reindexing && !chainman.ActiveChainstate().LoadGenesisBlock()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } diff --git a/src/protocol.cpp b/src/protocol.cpp index 0da160768d..0439d398c7 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -7,87 +7,6 @@ #include <common/system.h> -#include <atomic> - -namespace NetMsgType { -const char* VERSION = "version"; -const char* VERACK = "verack"; -const char* ADDR = "addr"; -const char* ADDRV2 = "addrv2"; -const char* SENDADDRV2 = "sendaddrv2"; -const char* INV = "inv"; -const char* GETDATA = "getdata"; -const char* MERKLEBLOCK = "merkleblock"; -const char* GETBLOCKS = "getblocks"; -const char* GETHEADERS = "getheaders"; -const char* TX = "tx"; -const char* HEADERS = "headers"; -const char* BLOCK = "block"; -const char* GETADDR = "getaddr"; -const char* MEMPOOL = "mempool"; -const char* PING = "ping"; -const char* PONG = "pong"; -const char* NOTFOUND = "notfound"; -const char* FILTERLOAD = "filterload"; -const char* FILTERADD = "filteradd"; -const char* FILTERCLEAR = "filterclear"; -const char* SENDHEADERS = "sendheaders"; -const char* FEEFILTER = "feefilter"; -const char* SENDCMPCT = "sendcmpct"; -const char* CMPCTBLOCK = "cmpctblock"; -const char* GETBLOCKTXN = "getblocktxn"; -const char* BLOCKTXN = "blocktxn"; -const char* GETCFILTERS = "getcfilters"; -const char* CFILTER = "cfilter"; -const char* GETCFHEADERS = "getcfheaders"; -const char* CFHEADERS = "cfheaders"; -const char* GETCFCHECKPT = "getcfcheckpt"; -const char* CFCHECKPT = "cfcheckpt"; -const char* WTXIDRELAY = "wtxidrelay"; -const char* SENDTXRCNCL = "sendtxrcncl"; -} // namespace NetMsgType - -/** All known message types. Keep this in the same order as the list of - * messages above and in protocol.h. - */ -const static std::vector<std::string> g_all_net_message_types{ - NetMsgType::VERSION, - NetMsgType::VERACK, - NetMsgType::ADDR, - NetMsgType::ADDRV2, - NetMsgType::SENDADDRV2, - NetMsgType::INV, - NetMsgType::GETDATA, - NetMsgType::MERKLEBLOCK, - NetMsgType::GETBLOCKS, - NetMsgType::GETHEADERS, - NetMsgType::TX, - NetMsgType::HEADERS, - NetMsgType::BLOCK, - NetMsgType::GETADDR, - NetMsgType::MEMPOOL, - NetMsgType::PING, - NetMsgType::PONG, - NetMsgType::NOTFOUND, - NetMsgType::FILTERLOAD, - NetMsgType::FILTERADD, - NetMsgType::FILTERCLEAR, - NetMsgType::SENDHEADERS, - NetMsgType::FEEFILTER, - NetMsgType::SENDCMPCT, - NetMsgType::CMPCTBLOCK, - NetMsgType::GETBLOCKTXN, - NetMsgType::BLOCKTXN, - NetMsgType::GETCFILTERS, - NetMsgType::CFILTER, - NetMsgType::GETCFHEADERS, - NetMsgType::CFHEADERS, - NetMsgType::GETCFCHECKPT, - NetMsgType::CFCHECKPT, - NetMsgType::WTXIDRELAY, - NetMsgType::SENDTXRCNCL, -}; - CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn) : pchMessageStart{pchMessageStartIn} { @@ -164,11 +83,6 @@ std::string CInv::ToString() const } } -const std::vector<std::string> &getAllNetMessageTypes() -{ - return g_all_net_message_types; -} - /** * Convert a service flag (NODE_*) to a human readable string. * It supports unknown service flags which will be returned as "UNKNOWN[...]". diff --git a/src/protocol.h b/src/protocol.h index fba08c6f55..fd7cfddf3b 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -55,106 +55,105 @@ public: /** * Bitcoin protocol message types. When adding new message types, don't forget - * to update allNetMessageTypes in protocol.cpp. + * to update ALL_NET_MESSAGE_TYPES below. */ namespace NetMsgType { - /** * The version message provides information about the transmitting node to the * receiving node at the beginning of a connection. */ -extern const char* VERSION; +inline constexpr const char* VERSION{"version"}; /** * The verack message acknowledges a previously-received version message, * informing the connecting node that it can begin to send other messages. */ -extern const char* VERACK; +inline constexpr const char* VERACK{"verack"}; /** * The addr (IP address) message relays connection information for peers on the * network. */ -extern const char* ADDR; +inline constexpr const char* ADDR{"addr"}; /** * The addrv2 message relays connection information for peers on the network just * like the addr message, but is extended to allow gossiping of longer node * addresses (see BIP155). */ -extern const char *ADDRV2; +inline constexpr const char* ADDRV2{"addrv2"}; /** * The sendaddrv2 message signals support for receiving ADDRV2 messages (BIP155). * It also implies that its sender can encode as ADDRV2 and would send ADDRV2 * instead of ADDR to a peer that has signaled ADDRV2 support by sending SENDADDRV2. */ -extern const char *SENDADDRV2; +inline constexpr const char* SENDADDRV2{"sendaddrv2"}; /** * The inv message (inventory message) transmits one or more inventories of * objects known to the transmitting peer. */ -extern const char* INV; +inline constexpr const char* INV{"inv"}; /** * The getdata message requests one or more data objects from another node. */ -extern const char* GETDATA; +inline constexpr const char* GETDATA{"getdata"}; /** * The merkleblock message is a reply to a getdata message which requested a * block using the inventory type MSG_MERKLEBLOCK. * @since protocol version 70001 as described by BIP37. */ -extern const char* MERKLEBLOCK; +inline constexpr const char* MERKLEBLOCK{"merkleblock"}; /** * The getblocks message requests an inv message that provides block header * hashes starting from a particular point in the block chain. */ -extern const char* GETBLOCKS; +inline constexpr const char* GETBLOCKS{"getblocks"}; /** * The getheaders message requests a headers message that provides block * headers starting from a particular point in the block chain. * @since protocol version 31800. */ -extern const char* GETHEADERS; +inline constexpr const char* GETHEADERS{"getheaders"}; /** * The tx message transmits a single transaction. */ -extern const char* TX; +inline constexpr const char* TX{"tx"}; /** * The headers message sends one or more block headers to a node which * previously requested certain headers with a getheaders message. * @since protocol version 31800. */ -extern const char* HEADERS; +inline constexpr const char* HEADERS{"headers"}; /** * The block message transmits a single serialized block. */ -extern const char* BLOCK; +inline constexpr const char* BLOCK{"block"}; /** * The getaddr message requests an addr message from the receiving node, * preferably one with lots of IP addresses of other receiving nodes. */ -extern const char* GETADDR; +inline constexpr const char* GETADDR{"getaddr"}; /** * The mempool message requests the TXIDs of transactions that the receiving * node has verified as valid but which have not yet appeared in a block. * @since protocol version 60002 as described by BIP35. * Only available with service bit NODE_BLOOM, see also BIP111. */ -extern const char* MEMPOOL; +inline constexpr const char* MEMPOOL{"mempool"}; /** * The ping message is sent periodically to help confirm that the receiving * peer is still connected. */ -extern const char* PING; +inline constexpr const char* PING{"ping"}; /** * The pong message replies to a ping message, proving to the pinging node that * the ponging node is still alive. * @since protocol version 60001 as described by BIP31. */ -extern const char* PONG; +inline constexpr const char* PONG{"pong"}; /** * The notfound message is a reply to a getdata message which requested an * object the receiving node does not have available for relay. * @since protocol version 70001. */ -extern const char* NOTFOUND; +inline constexpr const char* NOTFOUND{"notfound"}; /** * The filterload message tells the receiving peer to filter all relayed * transactions and requested merkle blocks through the provided filter. @@ -162,7 +161,7 @@ extern const char* NOTFOUND; * Only available with service bit NODE_BLOOM since protocol version * 70011 as described by BIP111. */ -extern const char* FILTERLOAD; +inline constexpr const char* FILTERLOAD{"filterload"}; /** * The filteradd message tells the receiving peer to add a single element to a * previously-set bloom filter, such as a new public key. @@ -170,7 +169,7 @@ extern const char* FILTERLOAD; * Only available with service bit NODE_BLOOM since protocol version * 70011 as described by BIP111. */ -extern const char* FILTERADD; +inline constexpr const char* FILTERADD{"filteradd"}; /** * The filterclear message tells the receiving peer to remove a previously-set * bloom filter. @@ -178,19 +177,19 @@ extern const char* FILTERADD; * Only available with service bit NODE_BLOOM since protocol version * 70011 as described by BIP111. */ -extern const char* FILTERCLEAR; +inline constexpr const char* FILTERCLEAR{"filterclear"}; /** * Indicates that a node prefers to receive new block announcements via a * "headers" message rather than an "inv". * @since protocol version 70012 as described by BIP130. */ -extern const char* SENDHEADERS; +inline constexpr const char* SENDHEADERS{"sendheaders"}; /** * The feefilter message tells the receiving peer not to inv us any txs * which do not meet the specified min fee rate. * @since protocol version 70013 as described by BIP133 */ -extern const char* FEEFILTER; +inline constexpr const char* FEEFILTER{"feefilter"}; /** * Contains a 1-byte bool and 8-byte LE version number. * Indicates that a node is willing to provide blocks via "cmpctblock" messages. @@ -198,36 +197,36 @@ extern const char* FEEFILTER; * "cmpctblock" message rather than an "inv", depending on message contents. * @since protocol version 70014 as described by BIP 152 */ -extern const char* SENDCMPCT; +inline constexpr const char* SENDCMPCT{"sendcmpct"}; /** * Contains a CBlockHeaderAndShortTxIDs object - providing a header and * list of "short txids". * @since protocol version 70014 as described by BIP 152 */ -extern const char* CMPCTBLOCK; +inline constexpr const char* CMPCTBLOCK{"cmpctblock"}; /** * Contains a BlockTransactionsRequest * Peer should respond with "blocktxn" message. * @since protocol version 70014 as described by BIP 152 */ -extern const char* GETBLOCKTXN; +inline constexpr const char* GETBLOCKTXN{"getblocktxn"}; /** * Contains a BlockTransactions. * Sent in response to a "getblocktxn" message. * @since protocol version 70014 as described by BIP 152 */ -extern const char* BLOCKTXN; +inline constexpr const char* BLOCKTXN{"blocktxn"}; /** * getcfilters requests compact filters for a range of blocks. * Only available with service bit NODE_COMPACT_FILTERS as described by * BIP 157 & 158. */ -extern const char* GETCFILTERS; +inline constexpr const char* GETCFILTERS{"getcfilters"}; /** * cfilter is a response to a getcfilters request containing a single compact * filter. */ -extern const char* CFILTER; +inline constexpr const char* CFILTER{"cfilter"}; /** * getcfheaders requests a compact filter header and the filter hashes for a * range of blocks, which can then be used to reconstruct the filter headers @@ -235,40 +234,76 @@ extern const char* CFILTER; * Only available with service bit NODE_COMPACT_FILTERS as described by * BIP 157 & 158. */ -extern const char* GETCFHEADERS; +inline constexpr const char* GETCFHEADERS{"getcfheaders"}; /** * cfheaders is a response to a getcfheaders request containing a filter header * and a vector of filter hashes for each subsequent block in the requested range. */ -extern const char* CFHEADERS; +inline constexpr const char* CFHEADERS{"cfheaders"}; /** * getcfcheckpt requests evenly spaced compact filter headers, enabling * parallelized download and validation of the headers between them. * Only available with service bit NODE_COMPACT_FILTERS as described by * BIP 157 & 158. */ -extern const char* GETCFCHECKPT; +inline constexpr const char* GETCFCHECKPT{"getcfcheckpt"}; /** * cfcheckpt is a response to a getcfcheckpt request containing a vector of * evenly spaced filter headers for blocks on the requested chain. */ -extern const char* CFCHECKPT; +inline constexpr const char* CFCHECKPT{"cfcheckpt"}; /** * Indicates that a node prefers to relay transactions via wtxid, rather than * txid. * @since protocol version 70016 as described by BIP 339. */ -extern const char* WTXIDRELAY; +inline constexpr const char* WTXIDRELAY{"wtxidrelay"}; /** * Contains a 4-byte version number and an 8-byte salt. * The salt is used to compute short txids needed for efficient * txreconciliation, as described by BIP 330. */ -extern const char* SENDTXRCNCL; +inline constexpr const char* SENDTXRCNCL{"sendtxrcncl"}; }; // namespace NetMsgType -/* Get a vector of all valid message types (see above) */ -const std::vector<std::string>& getAllNetMessageTypes(); +/** All known message types (see above). Keep this in the same order as the list of messages above. */ +inline const std::array ALL_NET_MESSAGE_TYPES{std::to_array<std::string>({ + NetMsgType::VERSION, + NetMsgType::VERACK, + NetMsgType::ADDR, + NetMsgType::ADDRV2, + NetMsgType::SENDADDRV2, + NetMsgType::INV, + NetMsgType::GETDATA, + NetMsgType::MERKLEBLOCK, + NetMsgType::GETBLOCKS, + NetMsgType::GETHEADERS, + NetMsgType::TX, + NetMsgType::HEADERS, + NetMsgType::BLOCK, + NetMsgType::GETADDR, + NetMsgType::MEMPOOL, + NetMsgType::PING, + NetMsgType::PONG, + NetMsgType::NOTFOUND, + NetMsgType::FILTERLOAD, + NetMsgType::FILTERADD, + NetMsgType::FILTERCLEAR, + NetMsgType::SENDHEADERS, + NetMsgType::FEEFILTER, + NetMsgType::SENDCMPCT, + NetMsgType::CMPCTBLOCK, + NetMsgType::GETBLOCKTXN, + NetMsgType::BLOCKTXN, + NetMsgType::GETCFILTERS, + NetMsgType::CFILTER, + NetMsgType::GETCFHEADERS, + NetMsgType::CFHEADERS, + NetMsgType::GETCFCHECKPT, + NetMsgType::CFCHECKPT, + NetMsgType::WTXIDRELAY, + NetMsgType::SENDTXRCNCL, +})}; /** nServices flags */ enum ServiceFlags : uint64_t { diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 11e1b4abb5..13e3c2dbe0 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -13,6 +13,7 @@ #include <secp256k1_schnorrsig.h> #include <span.h> #include <uint256.h> +#include <util/strencodings.h> #include <algorithm> #include <cassert> @@ -181,6 +182,17 @@ int ecdsa_signature_parse_der_lax(secp256k1_ecdsa_signature* sig, const unsigned return 1; } +/** Nothing Up My Sleeve (NUMS) point + * + * NUMS_H is a point with an unknown discrete logarithm, constructed by taking the sha256 of 'g' + * (uncompressed encoding), which happens to be a point on the curve. + * + * For an example script for calculating H, refer to the unit tests in + * ./test/functional/test_framework/crypto/secp256k1.py + */ +static const std::vector<unsigned char> NUMS_H_DATA{ParseHex("50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0")}; +const XOnlyPubKey XOnlyPubKey::NUMS_H{NUMS_H_DATA}; + XOnlyPubKey::XOnlyPubKey(Span<const unsigned char> bytes) { assert(bytes.size() == 32); diff --git a/src/pubkey.h b/src/pubkey.h index 15d7e7bc07..ae34ddd0af 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -233,6 +233,11 @@ private: uint256 m_keydata; public: + /** Nothing Up My Sleeve point H + * Used as an internal key for provably disabling the key path spend + * see BIP341 for more details */ + static const XOnlyPubKey NUMS_H; + /** Construct an empty x-only pubkey. */ XOnlyPubKey() = default; diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index 75e42e4c88..83a9010681 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -10,6 +10,7 @@ enum HTTPStatusCode { HTTP_OK = 200, + HTTP_NO_CONTENT = 204, HTTP_BAD_REQUEST = 400, HTTP_UNAUTHORIZED = 401, HTTP_FORBIDDEN = 403, diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index b7acd62ee3..d35782189e 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -26,6 +26,17 @@ * * 1.0 spec: http://json-rpc.org/wiki/specification * 1.2 spec: http://jsonrpc.org/historical/json-rpc-over-http.html + * + * If the server receives a request with the JSON-RPC 2.0 marker `{"jsonrpc": "2.0"}` + * then Bitcoin will respond with a strictly specified response. + * It will only return an HTTP error code if an actual HTTP error is encountered + * such as the endpoint is not found (404) or the request is not formatted correctly (500). + * Otherwise the HTTP code is always OK (200) and RPC errors will be included in the + * response body. + * + * 2.0 spec: https://www.jsonrpc.org/specification + * + * Also see http://www.simple-is-better.org/rpc/#differences-between-1-0-and-2-0 */ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id) @@ -37,24 +48,25 @@ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, return request; } -UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id) +UniValue JSONRPCReplyObj(UniValue result, UniValue error, std::optional<UniValue> id, JSONRPCVersion jsonrpc_version) { UniValue reply(UniValue::VOBJ); - if (!error.isNull()) - reply.pushKV("result", NullUniValue); - else - reply.pushKV("result", result); - reply.pushKV("error", error); - reply.pushKV("id", id); + // Add JSON-RPC version number field in v2 only. + if (jsonrpc_version == JSONRPCVersion::V2) reply.pushKV("jsonrpc", "2.0"); + + // Add both result and error fields in v1, even though one will be null. + // Omit the null field in v2. + if (error.isNull()) { + reply.pushKV("result", std::move(result)); + if (jsonrpc_version == JSONRPCVersion::V1_LEGACY) reply.pushKV("error", NullUniValue); + } else { + if (jsonrpc_version == JSONRPCVersion::V1_LEGACY) reply.pushKV("result", NullUniValue); + reply.pushKV("error", std::move(error)); + } + if (id.has_value()) reply.pushKV("id", std::move(id.value())); return reply; } -std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id) -{ - UniValue reply = JSONRPCReplyObj(result, error, id); - return reply.write() + "\n"; -} - UniValue JSONRPCError(int code, const std::string& message) { UniValue error(UniValue::VOBJ); @@ -171,7 +183,30 @@ void JSONRPCRequest::parse(const UniValue& valRequest) const UniValue& request = valRequest.get_obj(); // Parse id now so errors from here on will have the id - id = request.find_value("id"); + if (request.exists("id")) { + id = request.find_value("id"); + } else { + id = std::nullopt; + } + + // Check for JSON-RPC 2.0 (default 1.1) + m_json_version = JSONRPCVersion::V1_LEGACY; + const UniValue& jsonrpc_version = request.find_value("jsonrpc"); + if (!jsonrpc_version.isNull()) { + if (!jsonrpc_version.isStr()) { + throw JSONRPCError(RPC_INVALID_REQUEST, "jsonrpc field must be a string"); + } + // The "jsonrpc" key was added in the 2.0 spec, but some older documentation + // incorrectly included {"jsonrpc":"1.0"} in a request object, so we + // maintain that for backwards compatibility. + if (jsonrpc_version.get_str() == "1.0") { + m_json_version = JSONRPCVersion::V1_LEGACY; + } else if (jsonrpc_version.get_str() == "2.0") { + m_json_version = JSONRPCVersion::V2; + } else { + throw JSONRPCError(RPC_INVALID_REQUEST, "JSON-RPC version not supported"); + } + } // Parse method const UniValue& valMethod{request.find_value("method")}; diff --git a/src/rpc/request.h b/src/rpc/request.h index a682c58d96..e47f90af86 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -7,13 +7,18 @@ #define BITCOIN_RPC_REQUEST_H #include <any> +#include <optional> #include <string> #include <univalue.h> +enum class JSONRPCVersion { + V1_LEGACY, + V2 +}; + UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); -UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id); -std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id); +UniValue JSONRPCReplyObj(UniValue result, UniValue error, std::optional<UniValue> id, JSONRPCVersion jsonrpc_version); UniValue JSONRPCError(int code, const std::string& message); /** Generate a new RPC authentication cookie and write it to disk */ @@ -28,7 +33,7 @@ std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue& in); class JSONRPCRequest { public: - UniValue id; + std::optional<UniValue> id = UniValue::VNULL; std::string strMethod; UniValue params; enum Mode { EXECUTE, GET_HELP, GET_ARGS } mode = EXECUTE; @@ -36,8 +41,10 @@ public: std::string authUser; std::string peerAddr; std::any context; + JSONRPCVersion m_json_version = JSONRPCVersion::V1_LEGACY; void parse(const UniValue& valRequest); + [[nodiscard]] bool IsNotification() const { return !id.has_value() && m_json_version == JSONRPCVersion::V2; }; }; #endif // BITCOIN_RPC_REQUEST_H diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index a800451f4a..1ed406354a 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -360,36 +360,22 @@ bool IsDeprecatedRPCEnabled(const std::string& method) return find(enabled_methods.begin(), enabled_methods.end(), method) != enabled_methods.end(); } -static UniValue JSONRPCExecOne(JSONRPCRequest jreq, const UniValue& req) -{ - UniValue rpc_result(UniValue::VOBJ); - - try { - jreq.parse(req); - - UniValue result = tableRPC.execute(jreq); - rpc_result = JSONRPCReplyObj(result, NullUniValue, jreq.id); - } - catch (const UniValue& objError) - { - rpc_result = JSONRPCReplyObj(NullUniValue, objError, jreq.id); - } - catch (const std::exception& e) - { - rpc_result = JSONRPCReplyObj(NullUniValue, - JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); +UniValue JSONRPCExec(const JSONRPCRequest& jreq, bool catch_errors) +{ + UniValue result; + if (catch_errors) { + try { + result = tableRPC.execute(jreq); + } catch (UniValue& e) { + return JSONRPCReplyObj(NullUniValue, std::move(e), jreq.id, jreq.m_json_version); + } catch (const std::exception& e) { + return JSONRPCReplyObj(NullUniValue, JSONRPCError(RPC_MISC_ERROR, e.what()), jreq.id, jreq.m_json_version); + } + } else { + result = tableRPC.execute(jreq); } - return rpc_result; -} - -std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq) -{ - UniValue ret(UniValue::VARR); - for (unsigned int reqIdx = 0; reqIdx < vReq.size(); reqIdx++) - ret.push_back(JSONRPCExecOne(jreq, vReq[reqIdx])); - - return ret.write() + "\n"; + return JSONRPCReplyObj(std::move(result), NullUniValue, jreq.id, jreq.m_json_version); } /** diff --git a/src/rpc/server.h b/src/rpc/server.h index b8348e4aa6..5735aff821 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -179,6 +179,6 @@ extern CRPCTable tableRPC; void StartRPC(); void InterruptRPC(); void StopRPC(); -std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq); +UniValue JSONRPCExec(const JSONRPCRequest& jreq, bool catch_errors); #endif // BITCOIN_RPC_SERVER_H diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 9a7c731afe..f5a2e9eb63 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -175,7 +175,7 @@ std::string HelpExampleCliNamed(const std::string& methodname, const RPCArgList& std::string HelpExampleRpc(const std::string& methodname, const std::string& args) { - return "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", " + return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; } @@ -186,7 +186,7 @@ std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& params.pushKV(param.first, param.second); } - return "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", " + return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; } diff --git a/src/streams.cpp b/src/streams.cpp index 6921dad677..cdd36a86fe 100644 --- a/src/streams.cpp +++ b/src/streams.cpp @@ -21,6 +21,28 @@ std::size_t AutoFile::detail_fread(Span<std::byte> dst) } } +void AutoFile::seek(int64_t offset, int origin) +{ + if (IsNull()) { + throw std::ios_base::failure("AutoFile::seek: file handle is nullptr"); + } + if (std::fseek(m_file, offset, origin) != 0) { + throw std::ios_base::failure(feof() ? "AutoFile::seek: end of file" : "AutoFile::seek: fseek failed"); + } +} + +int64_t AutoFile::tell() +{ + if (IsNull()) { + throw std::ios_base::failure("AutoFile::tell: file handle is nullptr"); + } + int64_t r{std::ftell(m_file)}; + if (r < 0) { + throw std::ios_base::failure("AutoFile::tell: ftell failed"); + } + return r; +} + void AutoFile::read(Span<std::byte> dst) { if (detail_fread(dst) != dst.size()) { diff --git a/src/streams.h b/src/streams.h index bc04a2babd..57fc600646 100644 --- a/src/streams.h +++ b/src/streams.h @@ -435,6 +435,9 @@ public: /** Implementation detail, only used internally. */ std::size_t detail_fread(Span<std::byte> dst); + void seek(int64_t offset, int origin); + int64_t tell(); + // // Stream subset // diff --git a/src/sync.cpp b/src/sync.cpp index a8bdfc1dea..93c9194541 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -37,11 +37,11 @@ struct CLockLocation { const char* pszFile, int nLine, bool fTryIn, - const std::string& thread_name) + std::string&& thread_name) : fTry(fTryIn), mutexName(pszName), sourceFile(pszFile), - m_thread_name(thread_name), + m_thread_name(std::move(thread_name)), sourceLine(nLine) {} std::string ToString() const @@ -60,7 +60,7 @@ private: bool fTry; std::string mutexName; std::string sourceFile; - const std::string& m_thread_name; + const std::string m_thread_name; int sourceLine; }; diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index d7ac0bf823..efe0983698 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -35,20 +35,20 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) }; BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; // simulate adding a genesis block normally - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file // the block is found at offset 8 because there is an 8 byte serialization header // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. - FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + const FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; + blockman.UpdateBlockInfo(params->GenesisBlock(), 0, pos); // now simulate what happens after reindex for the first new block processed // the actual block contents don't matter, just that it's a block. // verify that the write position is at offset 0x12d. // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 - FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, nullptr)}; + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1)}; BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(TX_WITH_WITNESS(params->GenesisBlock())) + BLOCK_SERIALIZATION_HEADER_SIZE); } @@ -156,12 +156,11 @@ BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) // Blockstore is empty BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), 0); - // Write the first block; dbp=nullptr means this block doesn't already have a disk - // location, so allocate a free location and write it there. - FlatFilePos pos1{blockman.SaveBlockToDisk(block1, /*nHeight=*/1, /*dbp=*/nullptr)}; + // Write the first block to a new location. + FlatFilePos pos1{blockman.SaveBlockToDisk(block1, /*nHeight=*/1)}; // Write second block - FlatFilePos pos2{blockman.SaveBlockToDisk(block2, /*nHeight=*/2, /*dbp=*/nullptr)}; + FlatFilePos pos2{blockman.SaveBlockToDisk(block2, /*nHeight=*/2)}; // Two blocks in the file BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), (TEST_BLOCK_SIZE + BLOCK_SERIALIZATION_HEADER_SIZE) * 2); @@ -181,22 +180,19 @@ BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) BOOST_CHECK_EQUAL(read_block.nVersion, 2); } - // When FlatFilePos* dbp is given, SaveBlockToDisk() will not write or - // overwrite anything to the flat file block storage. It will, however, - // update the blockfile metadata. This is to facilitate reindexing - // when the user has the blocks on disk but the metadata is being rebuilt. + // During reindex, the flat file block storage will not be written to. + // UpdateBlockInfo will, however, update the blockfile metadata. // Verify this behavior by attempting (and failing) to write block 3 data // to block 2 location. CBlockFileInfo* block_data = blockman.GetBlockFileInfo(0); BOOST_CHECK_EQUAL(block_data->nBlocks, 2); - BOOST_CHECK(blockman.SaveBlockToDisk(block3, /*nHeight=*/3, /*dbp=*/&pos2) == pos2); + blockman.UpdateBlockInfo(block3, /*nHeight=*/3, /*pos=*/pos2); // Metadata is updated... BOOST_CHECK_EQUAL(block_data->nBlocks, 3); // ...but there are still only two blocks in the file BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), (TEST_BLOCK_SIZE + BLOCK_SERIALIZATION_HEADER_SIZE) * 2); // Block 2 was not overwritten: - // SaveBlockToDisk() did not call WriteBlockToDisk() because `FlatFilePos* dbp` was non-null blockman.ReadBlockFromDisk(read_block, pos2); BOOST_CHECK_EQUAL(read_block.nVersion, 2); } diff --git a/src/test/fuzz/miniscript.cpp b/src/test/fuzz/miniscript.cpp index f10007222c..7e71af7c44 100644 --- a/src/test/fuzz/miniscript.cpp +++ b/src/test/fuzz/miniscript.cpp @@ -309,9 +309,6 @@ const struct KeyComparator { // A dummy scriptsig to pass to VerifyScript (we always use Segwit v0). const CScript DUMMY_SCRIPTSIG; -//! Public key to be used as internal key for dummy Taproot spends. -const std::vector<unsigned char> NUMS_PK{ParseHex("50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0")}; - //! Construct a miniscript node as a shared_ptr. template<typename... Args> NodeRef MakeNodeRef(Args&&... args) { return miniscript::MakeNodeRef<CPubKey>(miniscript::internal::NoDupCheck{}, std::forward<Args>(args)...); @@ -1018,7 +1015,7 @@ CScript ScriptPubKey(MsCtx ctx, const CScript& script, TaprootBuilder& builder) // For Taproot outputs we always use a tree with a single script and a dummy internal key. builder.Add(0, script, TAPROOT_LEAF_TAPSCRIPT); - builder.Finalize(XOnlyPubKey{NUMS_PK}); + builder.Finalize(XOnlyPubKey::NUMS_H); return GetScriptForDestination(builder.GetOutput()); } diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index f6d82c3001..767238d103 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -21,13 +21,12 @@ namespace { -std::vector<std::string> g_all_messages; +auto g_all_messages = ALL_NET_MESSAGE_TYPES; void initialize_p2p_transport_serialization() { static ECC_Context ecc_context{}; SelectParams(ChainType::REGTEST); - g_all_messages = getAllNetMessageTypes(); std::sort(g_all_messages.begin(), g_all_messages.end()); } diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index a467fd5382..d10d9dafe8 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -37,7 +37,7 @@ void initialize_process_message() { if (const auto val{std::getenv("LIMIT_TO_MESSAGE_TYPE")}) { LIMIT_TO_MESSAGE_TYPE = val; - Assert(std::count(getAllNetMessageTypes().begin(), getAllNetMessageTypes().end(), LIMIT_TO_MESSAGE_TYPE)); // Unknown message type passed + Assert(std::count(ALL_NET_MESSAGE_TYPES.begin(), ALL_NET_MESSAGE_TYPES.end(), LIMIT_TO_MESSAGE_TYPE)); // Unknown message type passed } static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 86a8d17a76..aaf4ca4977 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -6,6 +6,7 @@ #include <common/system.h> #include <key_io.h> +#include <span.h> #include <streams.h> #include <test/util/random.h> #include <test/util/setup_common.h> @@ -364,4 +365,13 @@ BOOST_AUTO_TEST_CASE(key_ellswift) } } +BOOST_AUTO_TEST_CASE(bip341_test_h) +{ + std::vector<unsigned char> G_uncompressed = ParseHex("0479be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8"); + HashWriter hw; + hw.write(MakeByteSpan(G_uncompressed)); + XOnlyPubKey H{hw.GetSHA256()}; + BOOST_CHECK(XOnlyPubKey::NUMS_H == H); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index a3699f84b6..7e39e9e4de 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -288,9 +288,6 @@ public: } }; -//! Public key to be used as internal key for dummy Taproot spends. -const std::vector<unsigned char> NUMS_PK{ParseHex("50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0")}; - using Fragment = miniscript::Fragment; using NodeRef = miniscript::NodeRef<CPubKey>; using miniscript::operator"" _mst; @@ -330,7 +327,7 @@ CScript ScriptPubKey(miniscript::MiniscriptContext ctx, const CScript& script, T // For Taproot outputs we always use a tree with a single script and a dummy internal key. builder.Add(0, script, TAPROOT_LEAF_TAPSCRIPT); - builder.Finalize(XOnlyPubKey{NUMS_PK}); + builder.Finalize(XOnlyPubKey::NUMS_H); return GetScriptForDestination(builder.GetOutput()); } diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp index 58cbe9eb72..00bc1fdb6a 100644 --- a/src/test/net_peer_connection_tests.cpp +++ b/src/test/net_peer_connection_tests.cpp @@ -108,6 +108,12 @@ BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) AddPeer(id, nodes, *peerman, *connman, ConnectionType::BLOCK_RELAY, /*onion_peer=*/true); AddPeer(id, nodes, *peerman, *connman, ConnectionType::INBOUND); + // Add a CJDNS peer connection. + AddPeer(id, nodes, *peerman, *connman, ConnectionType::INBOUND, /*onion_peer=*/false, + /*address=*/"[fc00:3344:5566:7788:9900:aabb:ccdd:eeff]:1234"); + BOOST_CHECK(nodes.back()->IsInboundConn()); + BOOST_CHECK_EQUAL(nodes.back()->ConnectedThroughNetwork(), Network::NET_CJDNS); + BOOST_TEST_MESSAGE("Call AddNode() for all the peers"); for (auto node : connman->TestNodes()) { BOOST_CHECK(connman->AddNode({/*m_added_node=*/node->addr.ToStringAddrPort(), /*m_use_v2transport=*/true})); diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index acacb6257d..1c7d11d8a4 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -552,7 +552,7 @@ BOOST_AUTO_TEST_CASE(help_example) // test different argument types const RPCArgList& args = {{"foo", "bar"}, {"b", true}, {"n", 1}}; BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", args), "> bitcoin-cli -named test foo=bar b=true n=1\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); // test shell escape BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b'ar"}}), "> bitcoin-cli -named test foo='b'''ar'\n"); @@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(help_example) obj_value.pushKV("b", false); obj_value.pushKV("n", 1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", obj_value}}), "> bitcoin-cli -named test name='{\"foo\":\"bar\",\"b\":false,\"n\":1}'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); // test array params UniValue arr_value(UniValue::VARR); @@ -573,7 +573,7 @@ BOOST_AUTO_TEST_CASE(help_example) arr_value.push_back(false); arr_value.push_back(1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", arr_value}}), "> bitcoin-cli -named test name='[\"bar\",false,1]'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); // test types don't matter for shell BOOST_CHECK_EQUAL(HelpExampleCliNamed("foo", {{"arg", true}}), HelpExampleCliNamed("foo", {{"arg", "true"}})); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index e4142e203c..314b26609c 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1268,8 +1268,7 @@ BOOST_AUTO_TEST_CASE(sign_invalid_miniscript) const auto invalid_pubkey{ParseHex("173d36c8c9c9c9ffffffffffff0200000000021e1e37373721361818181818181e1e1e1e19000000000000000000b19292929292926b006c9b9b9292")}; TaprootBuilder builder; builder.Add(0, {invalid_pubkey}, 0xc0); - XOnlyPubKey nums{ParseHex("50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0")}; - builder.Finalize(nums); + builder.Finalize(XOnlyPubKey::NUMS_H); prev.vout.emplace_back(0, GetScriptForDestination(builder.GetOutput())); curr.vin.emplace_back(COutPoint{prev.GetHash(), 0}); sig_data.tr_spenddata = builder.GetSpendData(); diff --git a/src/test/util/random.h b/src/test/util/random.h index c910bd6a3a..18ab425e48 100644 --- a/src/test/util/random.h +++ b/src/test/util/random.h @@ -14,9 +14,8 @@ /** * This global and the helpers that use it are not thread-safe. * - * If thread-safety is needed, the global could be made thread_local (given - * that thread_local is supported on all architectures we support) or a - * per-thread instance could be used in the multi-threaded test. + * If thread-safety is needed, a per-thread instance could be + * used in the multi-threaded test. */ extern FastRandomContext g_insecure_rand_ctx; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index e9566cf50b..fd07931716 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -276,7 +276,7 @@ void ChainTestingSetup::LoadVerifyActivateChainstate() options.mempool = Assert(m_node.mempool.get()); options.block_tree_db_in_memory = m_block_tree_db_in_memory; options.coins_db_in_memory = m_coins_db_in_memory; - options.reindex = node::fReindex; + options.reindex = chainman.m_blockman.m_reindexing; options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index df5b1a4461..174052d5fa 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -11,8 +11,6 @@ #include <thread> #include <vector> -#include <config/bitcoin-config.h> // IWYU pragma: keep - #include <boost/test/unit_test.hpp> BOOST_AUTO_TEST_SUITE(util_threadnames_tests) @@ -52,11 +50,6 @@ std::set<std::string> RenameEnMasse(int num_threads) */ BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) { -#if !defined(HAVE_THREAD_LOCAL) - // This test doesn't apply to platforms where we don't have thread_local. - return; -#endif - std::set<std::string> names = RenameEnMasse(100); BOOST_CHECK_EQUAL(names.size(), 100U); diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index ea597dd05a..0249de37e3 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -4,6 +4,7 @@ #include <config/bitcoin-config.h> // IWYU pragma: keep +#include <cstring> #include <string> #include <thread> #include <utility> @@ -36,31 +37,30 @@ static void SetThreadName(const char* name) #endif } -// If we have thread_local, just keep thread ID and name in a thread_local -// global. -#if defined(HAVE_THREAD_LOCAL) - -static thread_local std::string g_thread_name; -const std::string& util::ThreadGetInternalName() { return g_thread_name; } +/** + * The name of the thread. We use char array instead of std::string to avoid + * complications with running a destructor when the thread exits. Avoid adding + * other thread_local variables. + * @see https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=278701 + */ +static thread_local char g_thread_name[128]{'\0'}; +std::string util::ThreadGetInternalName() { return g_thread_name; } //! Set the in-memory internal name for this thread. Does not affect the process //! name. -static void SetInternalName(std::string name) { g_thread_name = std::move(name); } - -// Without thread_local available, don't handle internal name at all. -#else - -static const std::string empty_string; -const std::string& util::ThreadGetInternalName() { return empty_string; } -static void SetInternalName(std::string name) { } -#endif +static void SetInternalName(const std::string& name) +{ + const size_t copy_bytes{std::min(sizeof(g_thread_name) - 1, name.length())}; + std::memcpy(g_thread_name, name.data(), copy_bytes); + g_thread_name[copy_bytes] = '\0'; +} -void util::ThreadRename(std::string&& name) +void util::ThreadRename(const std::string& name) { SetThreadName(("b-" + name).c_str()); - SetInternalName(std::move(name)); + SetInternalName(name); } -void util::ThreadSetInternalName(std::string&& name) +void util::ThreadSetInternalName(const std::string& name) { - SetInternalName(std::move(name)); + SetInternalName(name); } diff --git a/src/util/threadnames.h b/src/util/threadnames.h index 64b2689cf1..adca8c3000 100644 --- a/src/util/threadnames.h +++ b/src/util/threadnames.h @@ -12,14 +12,14 @@ namespace util { //! as its system thread name. //! @note Do not call this for the main thread, as this will interfere with //! UNIX utilities such as top and killall. Use ThreadSetInternalName instead. -void ThreadRename(std::string&&); +void ThreadRename(const std::string&); //! Set the internal (in-memory) name of the current thread only. -void ThreadSetInternalName(std::string&&); +void ThreadSetInternalName(const std::string&); //! Get the thread's internal (in-memory) name; used e.g. for identification in //! logging. -const std::string& ThreadGetInternalName(); +std::string ThreadGetInternalName(); } // namespace util diff --git a/src/validation.cpp b/src/validation.cpp index 90f5897b5f..d398ec7406 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -83,7 +83,6 @@ using node::BlockManager; using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; -using node::fReindex; using node::SnapshotMetadata; /** Time to wait between writing blocks/block index to disk. */ @@ -2642,7 +2641,7 @@ bool Chainstate::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); - if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { + if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !m_chainman.m_blockman.m_reindexing) { // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune @@ -3255,10 +3254,10 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* return true; } -static SynchronizationState GetSynchronizationState(bool init) +static SynchronizationState GetSynchronizationState(bool init, bool reindexing) { if (!init) return SynchronizationState::POST_INIT; - if (::fReindex) return SynchronizationState::INIT_REINDEX; + if (reindexing) return SynchronizationState::INIT_REINDEX; return SynchronizationState::INIT_DOWNLOAD; } @@ -3280,7 +3279,7 @@ static bool NotifyHeaderTip(ChainstateManager& chainman) LOCKS_EXCLUDED(cs_main) } // Send block tip changed notifications without cs_main if (fNotify) { - chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); + chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_reindexing), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3399,7 +3398,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< } // Always notify the UI if a new block tip was connected - if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd), *pindexNewTip))) { + if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_reindexing), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to // the caller so the caller could distinguish between @@ -3625,7 +3624,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // parameter indicating the source of the tip change so hooks can // distinguish user-initiated invalidateblock changes from other // changes. - (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload()), *to_mark_failed->pprev); + (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_reindexing), *to_mark_failed->pprev); } return true; } @@ -4264,7 +4263,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = IsInitialBlockDownload(); - GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_reindexing), height, timestamp, /*presync=*/true); if (initial_download) { int64_t blocks_left{(NodeClock::now() - NodeSeconds{std::chrono::seconds{timestamp}}) / GetConsensus().PowTargetSpacing()}; blocks_left = std::max<int64_t>(0, blocks_left); @@ -4345,10 +4344,16 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, dbp)}; - if (blockPos.IsNull()) { - state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); - return false; + FlatFilePos blockPos{}; + if (dbp) { + blockPos = *dbp; + m_blockman.UpdateBlockInfo(block, pindex->nHeight, blockPos); + } else { + blockPos = m_blockman.SaveBlockToDisk(block, pindex->nHeight); + if (blockPos.IsNull()) { + state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); + return false; + } } ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { @@ -4785,8 +4790,8 @@ bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); // Load block index from databases - bool needs_init = fReindex; - if (!fReindex) { + bool needs_init = m_blockman.m_reindexing; + if (!m_blockman.m_reindexing) { bool ret{m_blockman.LoadBlockIndexDB(SnapshotBlockhash())}; if (!ret) return false; @@ -4823,8 +4828,8 @@ bool ChainstateManager::LoadBlockIndex() if (needs_init) { // Everything here is for *new* reindex/DBs. Thus, though - // LoadBlockIndexDB may have set fReindex if we shut down - // mid-reindex previously, we don't check fReindex and + // LoadBlockIndexDB may have set m_reindexing if we shut down + // mid-reindex previously, we don't check m_reindexing and // instead only check it prior to LoadBlockIndexDB to set // needs_init. @@ -4848,7 +4853,7 @@ bool Chainstate::LoadGenesisBlock() try { const CBlock& block = params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, nullptr)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0)}; if (blockPos.IsNull()) { LogError("%s: writing genesis block to disk failed\n", __func__); return false; @@ -4969,7 +4974,7 @@ void ChainstateManager::LoadExternalBlockFile( } } - if (m_blockman.IsPruneMode() && !fReindex && pblock) { + if (m_blockman.IsPruneMode() && !m_blockman.m_reindexing && pblock) { // must update the tip for pruning to work while importing with -loadblock. // this is a tradeoff to conserve disk space at the expense of time // spent updating the tip to be able to prune. diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 38cca32f80..d82d8d4513 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -65,6 +65,8 @@ RecursiveMutex cs_db; std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. } // namespace +static constexpr auto REVERSE_BYTE_ORDER{std::endian::native == std::endian::little ? 4321 : 1234}; + bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const { return memcmp(value, &rhs.value, sizeof(value)) == 0; @@ -300,7 +302,11 @@ static Span<const std::byte> SpanFromDbt(const SafeDbt& dbt) } BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) : - WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb) + WalletDatabase(), + env(std::move(env)), + m_byteswap(options.require_format == DatabaseFormat::BERKELEY_SWAP), + m_filename(std::move(filename)), + m_max_log_mb(options.max_log_mb) { auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this)); assert(inserted.second); @@ -389,6 +395,10 @@ void BerkeleyDatabase::Open() } } + if (m_byteswap) { + pdb_temp->set_lorder(REVERSE_BYTE_ORDER); + } + ret = pdb_temp->open(nullptr, // Txn pointer fMockDb ? nullptr : strFile.c_str(), // Filename fMockDb ? strFile.c_str() : "main", // Logical db name @@ -521,6 +531,10 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) BerkeleyBatch db(*this, true); std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0); + if (m_byteswap) { + pdbCopy->set_lorder(REVERSE_BYTE_ORDER); + } + int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 630630ebe0..af0c78f0d9 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -147,6 +147,9 @@ public: /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ std::unique_ptr<Db> m_db; + // Whether to byteswap + bool m_byteswap; + fs::path m_filename; int64_t m_max_log_mb; diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index ea06767e9b..a5a5f8ec6f 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -16,6 +16,9 @@ #include <vector> namespace wallet { +bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } +bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } + std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) { std::vector<fs::path> paths; diff --git a/src/wallet/db.h b/src/wallet/db.h index 084fcadc24..b45076d10c 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -20,6 +20,12 @@ class ArgsManager; struct bilingual_str; namespace wallet { +// BytePrefix compares equality with other byte spans that begin with the same prefix. +struct BytePrefix { + Span<const std::byte> prefix; +}; +bool operator<(BytePrefix a, Span<const std::byte> b); +bool operator<(Span<const std::byte> a, BytePrefix b); class DatabaseCursor { @@ -177,6 +183,8 @@ public: enum class DatabaseFormat { BERKELEY, SQLITE, + BERKELEY_RO, + BERKELEY_SWAP, }; struct DatabaseOptions { diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 7a36910dc1..db2756e0ca 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -60,7 +60,13 @@ bool DumpWallet(const ArgsManager& args, WalletDatabase& db, bilingual_str& erro hasher << Span{line}; // Write out the file format - line = strprintf("%s,%s\n", "format", db.Format()); + std::string format = db.Format(); + // BDB files that are opened using BerkeleyRODatabase have it's format as "bdb_ro" + // We want to override that format back to "bdb" + if (format == "bdb_ro") { + format = "bdb"; + } + line = strprintf("%s,%s\n", "format", format); dump_file.write(line.data(), line.size()); hasher << Span{line}; @@ -180,6 +186,8 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: data_format = DatabaseFormat::BERKELEY; } else if (file_format == "sqlite") { data_format = DatabaseFormat::SQLITE; + } else if (file_format == "bdb_swap") { + data_format = DatabaseFormat::BERKELEY_SWAP; } else { error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format); return false; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 14e988ec1a..14d22bb54e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -85,8 +85,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DatabaseOptions().max_log_mb), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", !DatabaseOptions().use_shared_memory), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-swapbdbendian", "Swaps the internal endianness of BDB wallet databases (default: false)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); #else - argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"}); + argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb", "-swapbdbendian"}); #endif #ifdef USE_SQLITE diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp new file mode 100644 index 0000000000..09254a76ad --- /dev/null +++ b/src/wallet/migrate.cpp @@ -0,0 +1,784 @@ +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <compat/byteswap.h> +#include <crypto/common.h> // For ReadBE32 +#include <logging.h> +#include <streams.h> +#include <util/translation.h> +#include <wallet/migrate.h> + +#include <optional> +#include <variant> + +namespace wallet { +// Magic bytes in both endianness's +constexpr uint32_t BTREE_MAGIC = 0x00053162; // If the file endianness matches our system, we see this magic +constexpr uint32_t BTREE_MAGIC_OE = 0x62310500; // If the file endianness is the other one, we will see this magic + +// Subdatabase name +static const std::vector<std::byte> SUBDATABASE_NAME = {std::byte{'m'}, std::byte{'a'}, std::byte{'i'}, std::byte{'n'}}; + +enum class PageType : uint8_t { + /* + * BDB has several page types, most of which we do not use + * They are listed here for completeness, but commented out + * to avoid opening something unintended. + INVALID = 0, // Invalid page type + DUPLICATE = 1, // Duplicate. Deprecated and no longer used + HASH_UNSORTED = 2, // Hash pages. Deprecated. + RECNO_INTERNAL = 4, // Recno internal + RECNO_LEAF = 6, // Recno leaf + HASH_META = 8, // Hash metadata + QUEUE_META = 10, // Queue Metadata + QUEUE_DATA = 11, // Queue Data + DUPLICATE_LEAF = 12, // Off-page duplicate leaf + HASH_SORTED = 13, // Sorted hash page + */ + BTREE_INTERNAL = 3, // BTree internal + BTREE_LEAF = 5, // BTree leaf + OVERFLOW_DATA = 7, // Overflow + BTREE_META = 9, // BTree metadata +}; + +enum class RecordType : uint8_t { + KEYDATA = 1, + // DUPLICATE = 2, Unused as our databases do not support duplicate records + OVERFLOW_DATA = 3, + DELETE = 0x80, // Indicate this record is deleted. This is OR'd with the real type. +}; + +enum class BTreeFlags : uint32_t { + /* + * BTree databases have feature flags, but we do not use them except for + * subdatabases. The unused flags are included for completeness, but commented out + * to avoid accidental use. + DUP = 1, // Duplicates + RECNO = 2, // Recno tree + RECNUM = 4, // BTree: Maintain record counts + FIXEDLEN = 8, // Recno: fixed length records + RENUMBER = 0x10, // Recno: renumber on insert/delete + DUPSORT = 0x40, // Duplicates are sorted + COMPRESS = 0x80, // Compressed + */ + SUBDB = 0x20, // Subdatabases +}; + +/** Berkeley DB BTree metadata page layout */ +class MetaPage +{ +public: + uint32_t lsn_file; // Log Sequence Number file + uint32_t lsn_offset; // Log Sequence Number offset + uint32_t page_num; // Current page number + uint32_t magic; // Magic number + uint32_t version; // Version + uint32_t pagesize; // Page size + uint8_t encrypt_algo; // Encryption algorithm + PageType type; // Page type + uint8_t metaflags; // Meta-only flags + uint8_t unused1; // Unused + uint32_t free_list; // Free list page number + uint32_t last_page; // Page number of last page in db + uint32_t partitions; // Number of partitions + uint32_t key_count; // Cached key count + uint32_t record_count; // Cached record count + BTreeFlags flags; // Flags + std::array<std::byte, 20> uid; // 20 byte unique file ID + uint32_t unused2; // Unused + uint32_t minkey; // Minimum key + uint32_t re_len; // Recno: fixed length record length + uint32_t re_pad; // Recno: fixed length record pad + uint32_t root; // Root page number + char unused3[368]; // 92 * 4 bytes of unused space + uint32_t crypto_magic; // Crypto magic number + char trash[12]; // 3 * 4 bytes of trash space + unsigned char iv[20]; // Crypto IV + unsigned char chksum[16]; // Checksum + + bool other_endian; + uint32_t expected_page_num; + + MetaPage(uint32_t expected_page_num) : expected_page_num(expected_page_num) {} + MetaPage() = delete; + + template <typename Stream> + void Unserialize(Stream& s) + { + s >> lsn_file; + s >> lsn_offset; + s >> page_num; + s >> magic; + s >> version; + s >> pagesize; + s >> encrypt_algo; + + other_endian = magic == BTREE_MAGIC_OE; + + uint8_t uint8_type; + s >> uint8_type; + type = static_cast<PageType>(uint8_type); + + s >> metaflags; + s >> unused1; + s >> free_list; + s >> last_page; + s >> partitions; + s >> key_count; + s >> record_count; + + uint32_t uint32_flags; + s >> uint32_flags; + if (other_endian) { + uint32_flags = internal_bswap_32(uint32_flags); + } + flags = static_cast<BTreeFlags>(uint32_flags); + + s >> uid; + s >> unused2; + s >> minkey; + s >> re_len; + s >> re_pad; + s >> root; + s >> unused3; + s >> crypto_magic; + s >> trash; + s >> iv; + s >> chksum; + + if (other_endian) { + lsn_file = internal_bswap_32(lsn_file); + lsn_offset = internal_bswap_32(lsn_offset); + page_num = internal_bswap_32(page_num); + magic = internal_bswap_32(magic); + version = internal_bswap_32(version); + pagesize = internal_bswap_32(pagesize); + free_list = internal_bswap_32(free_list); + last_page = internal_bswap_32(last_page); + partitions = internal_bswap_32(partitions); + key_count = internal_bswap_32(key_count); + record_count = internal_bswap_32(record_count); + unused2 = internal_bswap_32(unused2); + minkey = internal_bswap_32(minkey); + re_len = internal_bswap_32(re_len); + re_pad = internal_bswap_32(re_pad); + root = internal_bswap_32(root); + crypto_magic = internal_bswap_32(crypto_magic); + } + + // Page number must match + if (page_num != expected_page_num) { + throw std::runtime_error("Meta page number mismatch"); + } + + // Check magic + if (magic != BTREE_MAGIC) { + throw std::runtime_error("Not a BDB file"); + } + + // Only version 9 is supported + if (version != 9) { + throw std::runtime_error("Unsupported BDB data file version number"); + } + + // Page size must be 512 <= pagesize <= 64k, and be a power of 2 + if (pagesize < 512 || pagesize > 65536 || (pagesize & (pagesize - 1)) != 0) { + throw std::runtime_error("Bad page size"); + } + + // Page type must be the btree type + if (type != PageType::BTREE_META) { + throw std::runtime_error("Unexpected page type, should be 9 (BTree Metadata)"); + } + + // Only supported meta-flag is subdatabase + if (flags != BTreeFlags::SUBDB) { + throw std::runtime_error("Unexpected database flags, should only be 0x20 (subdatabases)"); + } + } +}; + +/** General class for records in a BDB BTree database. Contains common fields. */ +class RecordHeader +{ +public: + uint16_t len; // Key/data item length + RecordType type; // Page type (BDB has this include a DELETE FLAG that we track separately) + bool deleted; // Whether the DELETE flag was set on type + + static constexpr size_t SIZE = 3; // The record header is 3 bytes + + bool other_endian; + + RecordHeader(bool other_endian) : other_endian(other_endian) {} + RecordHeader() = delete; + + template <typename Stream> + void Unserialize(Stream& s) + { + s >> len; + + uint8_t uint8_type; + s >> uint8_type; + type = static_cast<RecordType>(uint8_type & ~static_cast<uint8_t>(RecordType::DELETE)); + deleted = uint8_type & static_cast<uint8_t>(RecordType::DELETE); + + if (other_endian) { + len = internal_bswap_16(len); + } + } +}; + +/** Class for data in the record directly */ +class DataRecord +{ +public: + DataRecord(const RecordHeader& header) : m_header(header) {} + DataRecord() = delete; + + RecordHeader m_header; + + std::vector<std::byte> data; // Variable length key/data item + + template <typename Stream> + void Unserialize(Stream& s) + { + data.resize(m_header.len); + s.read(AsWritableBytes(Span(data.data(), data.size()))); + } +}; + +/** Class for records representing internal nodes of the BTree. */ +class InternalRecord +{ +public: + InternalRecord(const RecordHeader& header) : m_header(header) {} + InternalRecord() = delete; + + RecordHeader m_header; + + uint8_t unused; // Padding, unused + uint32_t page_num; // Page number of referenced page + uint32_t records; // Subtree record count + std::vector<std::byte> data; // Variable length key item + + static constexpr size_t FIXED_SIZE = 9; // Size of fixed data is 9 bytes + + template <typename Stream> + void Unserialize(Stream& s) + { + s >> unused; + s >> page_num; + s >> records; + + data.resize(m_header.len); + s.read(AsWritableBytes(Span(data.data(), data.size()))); + + if (m_header.other_endian) { + page_num = internal_bswap_32(page_num); + records = internal_bswap_32(records); + } + } +}; + +/** Class for records representing overflow records of the BTree. + * Overflow records point to a page which contains the data in the record. + * Those pages may point to further pages with the rest of the data if it does not fit + * in one page */ +class OverflowRecord +{ +public: + OverflowRecord(const RecordHeader& header) : m_header(header) {} + OverflowRecord() = delete; + + RecordHeader m_header; + + uint8_t unused2; // Padding, unused + uint32_t page_number; // Page number where data begins + uint32_t item_len; // Total length of item + + static constexpr size_t SIZE = 9; // Overflow record is always 9 bytes + + template <typename Stream> + void Unserialize(Stream& s) + { + s >> unused2; + s >> page_number; + s >> item_len; + + if (m_header.other_endian) { + page_number = internal_bswap_32(page_number); + item_len = internal_bswap_32(item_len); + } + } +}; + +/** A generic data page in the database. Contains fields common to all data pages. */ +class PageHeader +{ +public: + uint32_t lsn_file; // Log Sequence Number file + uint32_t lsn_offset; // Log Sequence Number offset + uint32_t page_num; // Current page number + uint32_t prev_page; // Previous page number + uint32_t next_page; // Next page number + uint16_t entries; // Number of items on the page + uint16_t hf_offset; // High free byte page offset + uint8_t level; // Btree page level + PageType type; // Page type + + static constexpr int64_t SIZE = 26; // The header is 26 bytes + + uint32_t expected_page_num; + bool other_endian; + + PageHeader(uint32_t page_num, bool other_endian) : expected_page_num(page_num), other_endian(other_endian) {} + PageHeader() = delete; + + template <typename Stream> + void Unserialize(Stream& s) + { + s >> lsn_file; + s >> lsn_offset; + s >> page_num; + s >> prev_page; + s >> next_page; + s >> entries; + s >> hf_offset; + s >> level; + + uint8_t uint8_type; + s >> uint8_type; + type = static_cast<PageType>(uint8_type); + + if (other_endian) { + lsn_file = internal_bswap_32(lsn_file); + lsn_offset = internal_bswap_32(lsn_offset); + page_num = internal_bswap_32(page_num); + prev_page = internal_bswap_32(prev_page); + next_page = internal_bswap_32(next_page); + entries = internal_bswap_16(entries); + hf_offset = internal_bswap_16(hf_offset); + } + + if (expected_page_num != page_num) { + throw std::runtime_error("Page number mismatch"); + } + if ((type != PageType::OVERFLOW_DATA && level < 1) || (type == PageType::OVERFLOW_DATA && level != 0)) { + throw std::runtime_error("Bad btree level"); + } + } +}; + +/** A page of records in the database */ +class RecordsPage +{ +public: + RecordsPage(const PageHeader& header) : m_header(header) {} + RecordsPage() = delete; + + PageHeader m_header; + + std::vector<uint16_t> indexes; + std::vector<std::variant<DataRecord, OverflowRecord>> records; + + template <typename Stream> + void Unserialize(Stream& s) + { + // Current position within the page + int64_t pos = PageHeader::SIZE; + + // Get the items + for (uint32_t i = 0; i < m_header.entries; ++i) { + // Get the index + uint16_t index; + s >> index; + if (m_header.other_endian) { + index = internal_bswap_16(index); + } + indexes.push_back(index); + pos += sizeof(uint16_t); + + // Go to the offset from the index + int64_t to_jump = index - pos; + if (to_jump < 0) { + throw std::runtime_error("Data record position not in page"); + } + s.ignore(to_jump); + + // Read the record + RecordHeader rec_hdr(m_header.other_endian); + s >> rec_hdr; + to_jump += RecordHeader::SIZE; + + switch (rec_hdr.type) { + case RecordType::KEYDATA: { + DataRecord record(rec_hdr); + s >> record; + records.emplace_back(record); + to_jump += rec_hdr.len; + break; + } + case RecordType::OVERFLOW_DATA: { + OverflowRecord record(rec_hdr); + s >> record; + records.emplace_back(record); + to_jump += OverflowRecord::SIZE; + break; + } + default: + throw std::runtime_error("Unknown record type in records page"); + } + + // Go back to the indexes + s.seek(-to_jump, SEEK_CUR); + } + } +}; + +/** A page containing overflow data */ +class OverflowPage +{ +public: + OverflowPage(const PageHeader& header) : m_header(header) {} + OverflowPage() = delete; + + PageHeader m_header; + + // BDB overloads some page fields to store overflow page data + // hf_offset contains the length of the overflow data stored on this page + // entries contains a reference count for references to this item + + // The overflow data itself. Begins immediately following header + std::vector<std::byte> data; + + template <typename Stream> + void Unserialize(Stream& s) + { + data.resize(m_header.hf_offset); + s.read(AsWritableBytes(Span(data.data(), data.size()))); + } +}; + +/** A page of records in the database */ +class InternalPage +{ +public: + InternalPage(const PageHeader& header) : m_header(header) {} + InternalPage() = delete; + + PageHeader m_header; + + std::vector<uint16_t> indexes; + std::vector<InternalRecord> records; + + template <typename Stream> + void Unserialize(Stream& s) + { + // Current position within the page + int64_t pos = PageHeader::SIZE; + + // Get the items + for (uint32_t i = 0; i < m_header.entries; ++i) { + // Get the index + uint16_t index; + s >> index; + if (m_header.other_endian) { + index = internal_bswap_16(index); + } + indexes.push_back(index); + pos += sizeof(uint16_t); + + // Go to the offset from the index + int64_t to_jump = index - pos; + if (to_jump < 0) { + throw std::runtime_error("Internal record position not in page"); + } + s.ignore(to_jump); + + // Read the record + RecordHeader rec_hdr(m_header.other_endian); + s >> rec_hdr; + to_jump += RecordHeader::SIZE; + + if (rec_hdr.type != RecordType::KEYDATA) { + throw std::runtime_error("Unknown record type in internal page"); + } + InternalRecord record(rec_hdr); + s >> record; + records.emplace_back(record); + to_jump += InternalRecord::FIXED_SIZE + rec_hdr.len; + + // Go back to the indexes + s.seek(-to_jump, SEEK_CUR); + } + } +}; + +static void SeekToPage(AutoFile& s, uint32_t page_num, uint32_t page_size) +{ + int64_t pos = int64_t{page_num} * page_size; + s.seek(pos, SEEK_SET); +} + +void BerkeleyRODatabase::Open() +{ + // Open the file + FILE* file = fsbridge::fopen(m_filepath, "rb"); + AutoFile db_file(file); + if (db_file.IsNull()) { + throw std::runtime_error("BerkeleyRODatabase: Failed to open database file"); + } + + uint32_t page_size = 4096; // Default page size + + // Read the outer metapage + // Expected page number is 0 + MetaPage outer_meta(0); + db_file >> outer_meta; + page_size = outer_meta.pagesize; + + // Verify the size of the file is a multiple of the page size + db_file.seek(0, SEEK_END); + int64_t size = db_file.tell(); + + // Since BDB stores everything in a page, the file size should be a multiple of the page size; + // However, BDB doesn't actually check that this is the case, and enforcing this check results + // in us rejecting a database that BDB would not, so this check needs to be excluded. + // This is left commented out as a reminder to not accidentally implement this in the future. + // if (size % page_size != 0) { + // throw std::runtime_error("File size is not a multiple of page size"); + // } + + // Check the last page number + uint32_t expected_last_page = (size / page_size) - 1; + if (outer_meta.last_page != expected_last_page) { + throw std::runtime_error("Last page number could not fit in file"); + } + + // Make sure encryption is disabled + if (outer_meta.encrypt_algo != 0) { + throw std::runtime_error("BDB builtin encryption is not supported"); + } + + // Check all Log Sequence Numbers (LSN) point to file 0 and offset 1 which indicates that the LSNs were + // reset and that the log files are not necessary to get all of the data in the database. + for (uint32_t i = 0; i < outer_meta.last_page; ++i) { + // The LSN is composed of 2 32-bit ints, the first is a file id, the second an offset + // It will always be the first 8 bytes of a page, so we deserialize it directly for every page + uint32_t file; + uint32_t offset; + SeekToPage(db_file, i, page_size); + db_file >> file >> offset; + if (outer_meta.other_endian) { + file = internal_bswap_32(file); + offset = internal_bswap_32(offset); + } + if (file != 0 || offset != 1) { + throw std::runtime_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support"); + } + } + + // Read the root page + SeekToPage(db_file, outer_meta.root, page_size); + PageHeader header(outer_meta.root, outer_meta.other_endian); + db_file >> header; + if (header.type != PageType::BTREE_LEAF) { + throw std::runtime_error("Unexpected outer database root page type"); + } + if (header.entries != 2) { + throw std::runtime_error("Unexpected number of entries in outer database root page"); + } + RecordsPage page(header); + db_file >> page; + + // First record should be the string "main" + if (!std::holds_alternative<DataRecord>(page.records.at(0)) || std::get<DataRecord>(page.records.at(0)).data != SUBDATABASE_NAME) { + throw std::runtime_error("Subdatabase has an unexpected name"); + } + // Check length of page number for subdatabase location + if (!std::holds_alternative<DataRecord>(page.records.at(1)) || std::get<DataRecord>(page.records.at(1)).m_header.len != 4) { + throw std::runtime_error("Subdatabase page number has unexpected length"); + } + + // Read subdatabase page number + // It is written as a big endian 32 bit number + uint32_t main_db_page = ReadBE32(UCharCast(std::get<DataRecord>(page.records.at(1)).data.data())); + + // The main database is in a page that doesn't exist + if (main_db_page > outer_meta.last_page) { + throw std::runtime_error("Page number is greater than database last page"); + } + + // Read the inner metapage + SeekToPage(db_file, main_db_page, page_size); + MetaPage inner_meta(main_db_page); + db_file >> inner_meta; + + if (inner_meta.pagesize != page_size) { + throw std::runtime_error("Unexpected page size"); + } + + if (inner_meta.last_page > outer_meta.last_page) { + throw std::runtime_error("Subdatabase last page is greater than database last page"); + } + + // Make sure encryption is disabled + if (inner_meta.encrypt_algo != 0) { + throw std::runtime_error("BDB builtin encryption is not supported"); + } + + // Do a DFS through the BTree, starting at root + std::vector<uint32_t> pages{inner_meta.root}; + while (pages.size() > 0) { + uint32_t curr_page = pages.back(); + // It turns out BDB completely ignores this last_page field and doesn't actually update it to the correct + // last page. While we should be checking this, we can't. + // This is left commented out as a reminder to not accidentally implement this in the future. + // if (curr_page > inner_meta.last_page) { + // throw std::runtime_error("Page number is greater than subdatabase last page"); + // } + pages.pop_back(); + SeekToPage(db_file, curr_page, page_size); + PageHeader header(curr_page, inner_meta.other_endian); + db_file >> header; + switch (header.type) { + case PageType::BTREE_INTERNAL: { + InternalPage int_page(header); + db_file >> int_page; + for (const InternalRecord& rec : int_page.records) { + if (rec.m_header.deleted) continue; + pages.push_back(rec.page_num); + } + break; + } + case PageType::BTREE_LEAF: { + RecordsPage rec_page(header); + db_file >> rec_page; + if (rec_page.records.size() % 2 != 0) { + // BDB stores key value pairs in consecutive records, thus an odd number of records is unexpected + throw std::runtime_error("Records page has odd number of records"); + } + bool is_key = true; + std::vector<std::byte> key; + for (const std::variant<DataRecord, OverflowRecord>& rec : rec_page.records) { + std::vector<std::byte> data; + if (const DataRecord* drec = std::get_if<DataRecord>(&rec)) { + if (drec->m_header.deleted) continue; + data = drec->data; + } else if (const OverflowRecord* orec = std::get_if<OverflowRecord>(&rec)) { + if (orec->m_header.deleted) continue; + uint32_t next_page = orec->page_number; + while (next_page != 0) { + SeekToPage(db_file, next_page, page_size); + PageHeader opage_header(next_page, inner_meta.other_endian); + db_file >> opage_header; + if (opage_header.type != PageType::OVERFLOW_DATA) { + throw std::runtime_error("Bad overflow record page type"); + } + OverflowPage opage(opage_header); + db_file >> opage; + data.insert(data.end(), opage.data.begin(), opage.data.end()); + next_page = opage_header.next_page; + } + } + + if (is_key) { + key = data; + } else { + m_records.emplace(SerializeData{key.begin(), key.end()}, SerializeData{data.begin(), data.end()}); + key.clear(); + } + is_key = !is_key; + } + break; + } + default: + throw std::runtime_error("Unexpected page type"); + } + } +} + +std::unique_ptr<DatabaseBatch> BerkeleyRODatabase::MakeBatch(bool flush_on_close) +{ + return std::make_unique<BerkeleyROBatch>(*this); +} + +bool BerkeleyRODatabase::Backup(const std::string& dest) const +{ + fs::path src(m_filepath); + fs::path dst(fs::PathFromString(dest)); + + if (fs::is_directory(dst)) { + dst = BDBDataFile(dst); + } + try { + if (fs::exists(dst) && fs::equivalent(src, dst)) { + LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(dst)); + return false; + } + + fs::copy_file(src, dst, fs::copy_options::overwrite_existing); + LogPrintf("copied %s to %s\n", fs::PathToString(m_filepath), fs::PathToString(dst)); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", fs::PathToString(m_filepath), fs::PathToString(dst), fsbridge::get_filesystem_error_message(e)); + return false; + } +} + +bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value) +{ + SerializeData key_data{key.begin(), key.end()}; + const auto it{m_database.m_records.find(key_data)}; + if (it == m_database.m_records.end()) { + return false; + } + auto val = it->second; + value.clear(); + value.write(Span(val)); + return true; +} + +bool BerkeleyROBatch::HasKey(DataStream&& key) +{ + SerializeData key_data{key.begin(), key.end()}; + return m_database.m_records.count(key_data) > 0; +} + +BerkeleyROCursor::BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix) + : m_database(database) +{ + std::tie(m_cursor, m_cursor_end) = m_database.m_records.equal_range(BytePrefix{prefix}); +} + +DatabaseCursor::Status BerkeleyROCursor::Next(DataStream& ssKey, DataStream& ssValue) +{ + if (m_cursor == m_cursor_end) { + return DatabaseCursor::Status::DONE; + } + ssKey.write(Span(m_cursor->first)); + ssValue.write(Span(m_cursor->second)); + m_cursor++; + return DatabaseCursor::Status::MORE; +} + +std::unique_ptr<DatabaseCursor> BerkeleyROBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + return std::make_unique<BerkeleyROCursor>(m_database, prefix); +} + +std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) +{ + fs::path data_file = BDBDataFile(path); + try { + std::unique_ptr<BerkeleyRODatabase> db = std::make_unique<BerkeleyRODatabase>(data_file); + status = DatabaseStatus::SUCCESS; + return db; + } catch (const std::runtime_error& e) { + error.original = e.what(); + status = DatabaseStatus::FAILED_LOAD; + return nullptr; + } +} +} // namespace wallet diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h new file mode 100644 index 0000000000..e4826450af --- /dev/null +++ b/src/wallet/migrate.h @@ -0,0 +1,124 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_MIGRATE_H +#define BITCOIN_WALLET_MIGRATE_H + +#include <wallet/db.h> + +#include <optional> + +namespace wallet { + +using BerkeleyROData = std::map<SerializeData, SerializeData, std::less<>>; + +/** + * A class representing a BerkeleyDB file from which we can only read records. + * This is used only for migration of legacy to descriptor wallets + */ +class BerkeleyRODatabase : public WalletDatabase +{ +private: + const fs::path m_filepath; + +public: + /** Create DB handle */ + BerkeleyRODatabase(const fs::path& filepath, bool open = true) : WalletDatabase(), m_filepath(filepath) + { + if (open) Open(); + } + ~BerkeleyRODatabase(){}; + + BerkeleyROData m_records; + + /** Open the database if it is not already opened. */ + void Open() override; + + /** Indicate the a new database user has began using the database. Increments m_refcount */ + void AddRef() override {} + /** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */ + void RemoveRef() override {} + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip = nullptr) override { return false; } + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest) const override; + + /** Make sure all changes are flushed to database file. + */ + void Flush() override {} + /** Flush to the database file and close the database. + * Also close the environment if no other databases are open in it. + */ + void Close() override {} + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + bool PeriodicFlush() override { return false; } + + void IncrementUpdateCounter() override {} + + void ReloadDbEnv() override {} + + /** Return path to main database file for logs and error messages. */ + std::string Filename() override { return fs::PathToString(m_filepath); } + + std::string Format() override { return "bdb_ro"; } + + /** Make a DatabaseBatch connected to this database */ + std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override; +}; + +class BerkeleyROCursor : public DatabaseCursor +{ +private: + const BerkeleyRODatabase& m_database; + BerkeleyROData::const_iterator m_cursor; + BerkeleyROData::const_iterator m_cursor_end; + +public: + explicit BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix = {}); + ~BerkeleyROCursor() {} + + Status Next(DataStream& key, DataStream& value) override; +}; + +/** RAII class that provides access to a BerkeleyRODatabase */ +class BerkeleyROBatch : public DatabaseBatch +{ +private: + const BerkeleyRODatabase& m_database; + + bool ReadKey(DataStream&& key, DataStream& value) override; + // WriteKey returns true since various automatic upgrades for older wallets will expect writing to not fail. + // It is okay for this batch type to not actually write anything as those automatic upgrades will occur again after migration. + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; } + bool EraseKey(DataStream&& key) override { return false; } + bool HasKey(DataStream&& key) override; + bool ErasePrefix(Span<const std::byte> prefix) override { return false; } + +public: + explicit BerkeleyROBatch(const BerkeleyRODatabase& database) : m_database(database) {} + ~BerkeleyROBatch() {} + + BerkeleyROBatch(const BerkeleyROBatch&) = delete; + BerkeleyROBatch& operator=(const BerkeleyROBatch&) = delete; + + void Flush() override {} + void Close() override {} + + std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<BerkeleyROCursor>(m_database); } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; + bool TxnBegin() override { return false; } + bool TxnCommit() override { return false; } + bool TxnAbort() override { return false; } +}; + +//! Return object giving access to Berkeley Read Only database at specified path. +std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); +} // namespace wallet + +#endif // BITCOIN_WALLET_MIGRATE_H diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 438dfceb7f..2fac356263 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -16,6 +16,7 @@ #ifdef USE_SQLITE #include <wallet/sqlite.h> #endif +#include <wallet/migrate.h> #include <wallet/test/util.h> #include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS @@ -132,6 +133,8 @@ static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path bilingual_str error; #ifdef USE_BDB dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error)); + // Needs BDB to make the DB to read + dbs.emplace_back(std::make_unique<BerkeleyRODatabase>(BDBDataFile(path_root / "bdb"), /*open=*/false)); #endif #ifdef USE_SQLITE dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error)); @@ -146,11 +149,16 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) for (const auto& database : TestDatabases(m_path_root)) { std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"}; - // Write elements to it std::unique_ptr<DatabaseBatch> handler = Assert(database)->MakeBatch(); - for (unsigned int i = 0; i < 10; i++) { - for (const auto& prefix : prefixes) { - BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + if (dynamic_cast<BerkeleyRODatabase*>(database.get())) { + // For BerkeleyRO, open the file now. This must happen after BDB has written to the file + database->Open(); + } else { + // Write elements to it if not berkeleyro + for (unsigned int i = 0; i < 10; i++) { + for (const auto& prefix : prefixes) { + BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + } } } @@ -178,6 +186,8 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) // Let's now read it once more, it should return DONE BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE); } + handler.reset(); + database->Close(); } } @@ -197,13 +207,23 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test) ffs{StringData("\xff\xffsuffix"), StringData("ffs")}; for (const auto& database : TestDatabases(m_path_root)) { std::unique_ptr<DatabaseBatch> batch = database->MakeBatch(); - for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { - batch->Write(Span{k}, Span{v}); + + if (dynamic_cast<BerkeleyRODatabase*>(database.get())) { + // For BerkeleyRO, open the file now. This must happen after BDB has written to the file + database->Open(); + } else { + // Write elements to it if not berkeleyro + for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { + batch->Write(Span{k}, Span{v}); + } } + CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs}); CheckPrefix(*batch, StringBytes("prefix"), {p, ps}); CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs}); CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs}); + batch.reset(); + database->Close(); } } @@ -213,6 +233,10 @@ BOOST_AUTO_TEST_CASE(db_availability_after_write_error) // To simulate the behavior, record overwrites are disallowed, and the test verifies // that the database remains active after failing to store an existing record. for (const auto& database : TestDatabases(m_path_root)) { + if (dynamic_cast<BerkeleyRODatabase*>(database.get())) { + // Skip this test if BerkeleyRO + continue; + } // Write original record std::unique_ptr<DatabaseBatch> batch = database->MakeBatch(); std::string key = "key"; @@ -241,6 +265,10 @@ BOOST_AUTO_TEST_CASE(erase_prefix) auto make_key = [](std::string type, std::string id) { return std::make_pair(type, id); }; for (const auto& database : TestDatabases(m_path_root)) { + if (dynamic_cast<BerkeleyRODatabase*>(database.get())) { + // Skip this test if BerkeleyRO + continue; + } std::unique_ptr<DatabaseBatch> batch = database->MakeBatch(); // Write two entries with the same key type prefix, a third one with a different prefix diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp new file mode 100644 index 0000000000..24ef75f791 --- /dev/null +++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <config/bitcoin-config.h> // IWYU pragma: keep +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <util/fs.h> +#include <util/time.h> +#include <util/translation.h> +#include <wallet/bdb.h> +#include <wallet/db.h> +#include <wallet/dump.h> +#include <wallet/migrate.h> + +#include <fstream> +#include <iostream> + +using wallet::DatabaseOptions; +using wallet::DatabaseStatus; + +namespace { +TestingSetup* g_setup; +} // namespace + +void initialize_wallet_bdb_parser() +{ + static auto testing_setup = MakeNoLogFileContext<TestingSetup>(); + g_setup = testing_setup.get(); +} + +FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) +{ + const auto wallet_path = g_setup->m_args.GetDataDirNet() / "fuzzed_wallet.dat"; + + { + AutoFile outfile{fsbridge::fopen(wallet_path, "wb")}; + outfile << Span{buffer}; + } + + const DatabaseOptions options{}; + DatabaseStatus status; + bilingual_str error; + + fs::path bdb_ro_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb_ro.dump"}; + if (fs::exists(bdb_ro_dumpfile)) { // Writing into an existing dump file will throw an exception + remove(bdb_ro_dumpfile); + } + g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_ro_dumpfile)); + +#ifdef USE_BDB + bool bdb_ro_err = false; + bool bdb_ro_pgno_err = false; +#endif + auto db{MakeBerkeleyRODatabase(wallet_path, options, status, error)}; + if (db) { + assert(DumpWallet(g_setup->m_args, *db, error)); + } else { +#ifdef USE_BDB + bdb_ro_err = true; +#endif + if (error.original == "AutoFile::ignore: end of file: iostream error" || + error.original == "AutoFile::read: end of file: iostream error" || + error.original == "Not a BDB file" || + error.original == "Unsupported BDB data file version number" || + error.original == "Unexpected page type, should be 9 (BTree Metadata)" || + error.original == "Unexpected database flags, should only be 0x20 (subdatabases)" || + error.original == "Unexpected outer database root page type" || + error.original == "Unexpected number of entries in outer database root page" || + error.original == "Subdatabase has an unexpected name" || + error.original == "Subdatabase page number has unexpected length" || + error.original == "Unexpected inner database page type" || + error.original == "Unknown record type in records page" || + error.original == "Unknown record type in internal page" || + error.original == "Unexpected page size" || + error.original == "Unexpected page type" || + error.original == "Page number mismatch" || + error.original == "Bad btree level" || + error.original == "Bad page size" || + error.original == "File size is not a multiple of page size" || + error.original == "Meta page number mismatch") { + // Do nothing + } else if (error.original == "Subdatabase last page is greater than database last page" || + error.original == "Page number is greater than database last page" || + error.original == "Page number is greater than subdatabase last page" || + error.original == "Last page number could not fit in file") { +#ifdef USE_BDB + bdb_ro_pgno_err = true; +#endif + } else { + throw std::runtime_error(error.original); + } + } + +#ifdef USE_BDB + // Try opening with BDB + fs::path bdb_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb.dump"}; + if (fs::exists(bdb_dumpfile)) { // Writing into an existing dump file will throw an exception + remove(bdb_dumpfile); + } + g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_dumpfile)); + + try { + auto db{MakeBerkeleyDatabase(wallet_path, options, status, error)}; + if (bdb_ro_err && !db) { + return; + } + assert(db); + if (bdb_ro_pgno_err) { + // BerkeleyRO will throw on opening for errors involving bad page numbers, but BDB does not. + // Ignore those. + return; + } + assert(!bdb_ro_err); + assert(DumpWallet(g_setup->m_args, *db, error)); + } catch (const std::runtime_error& e) { + if (bdb_ro_err) return; + throw e; + } + + // Make sure the dumpfiles match + if (fs::exists(bdb_ro_dumpfile) && fs::exists(bdb_dumpfile)) { + std::ifstream bdb_ro_dump(bdb_ro_dumpfile, std::ios_base::binary | std::ios_base::in); + std::ifstream bdb_dump(bdb_dumpfile, std::ios_base::binary | std::ios_base::in); + assert(std::equal( + std::istreambuf_iterator<char>(bdb_ro_dump.rdbuf()), + std::istreambuf_iterator<char>(), + std::istreambuf_iterator<char>(bdb_dump.rdbuf()))); + } +#endif +} diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 49d206f409..b21a9a601d 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -93,11 +93,6 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type) return *Assert(w.GetNewDestination(output_type, "")); } -// BytePrefix compares equality with other byte spans that begin with the same prefix. -struct BytePrefix { Span<const std::byte> prefix; }; -bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } -bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } - MockableCursor::MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix) { m_pass = pass; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 45f69f52d1..8a79cf730b 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -373,7 +373,12 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& uint64_t wallet_creation_flags = options.create_flags; const SecureString& passphrase = options.create_passphrase; + ArgsManager& args = *Assert(context.args); + if (wallet_creation_flags & WALLET_FLAG_DESCRIPTORS) options.require_format = DatabaseFormat::SQLITE; + else if (args.GetBoolArg("-swapbdbendian", false)) { + options.require_format = DatabaseFormat::BERKELEY_SWAP; + } // Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 3ba43cdb73..f34fcfc3fd 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -21,6 +21,7 @@ #ifdef USE_BDB #include <wallet/bdb.h> #endif +#include <wallet/migrate.h> #ifdef USE_SQLITE #include <wallet/sqlite.h> #endif @@ -1387,6 +1388,11 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas return nullptr; } + // If BERKELEY was the format, then change the format from BERKELEY to BERKELEY_RO + if (format && options.require_format && format == DatabaseFormat::BERKELEY && options.require_format == DatabaseFormat::BERKELEY_RO) { + format = DatabaseFormat::BERKELEY_RO; + } + // A db already exists so format is set, but options also specifies the format, so make sure they agree if (format && options.require_format && format != options.require_format) { error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", fs::PathToString(path))); @@ -1420,6 +1426,10 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas } } + if (format == DatabaseFormat::BERKELEY_RO) { + return MakeBerkeleyRODatabase(path, options, status, error); + } + #ifdef USE_BDB if constexpr (true) { return MakeBerkeleyDatabase(path, options, status, error); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 7a1930fd31..10785ad354 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -192,6 +192,11 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) ReadDatabaseArgs(args, options); options.require_existing = true; DatabaseStatus status; + + if (args.GetBoolArg("-withinternalbdb", false) && IsBDBFile(BDBDataFile(path))) { + options.require_format = DatabaseFormat::BERKELEY_RO; + } + bilingual_str error; std::unique_ptr<WalletDatabase> database = MakeDatabase(path, options, status, error); if (!database) { diff --git a/test/functional/feature_framework_unit_tests.py b/test/functional/feature_framework_unit_tests.py index c9754e083c..f03f084bed 100755 --- a/test/functional/feature_framework_unit_tests.py +++ b/test/functional/feature_framework_unit_tests.py @@ -25,6 +25,7 @@ TEST_FRAMEWORK_MODULES = [ "crypto.muhash", "crypto.poly1305", "crypto.ripemd160", + "crypto.secp256k1", "script", "segwit_addr", "wallet_util", diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index c5eeaf66e0..739b9b9bb9 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -28,7 +28,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [ [ - "-maxorphantx=1000", "-limitancestorcount=50", "-limitancestorsize=101", "-limitdescendantcount=200", diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index e873e2da0b..b08ca42796 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -4,22 +4,80 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" +import json import os -from test_framework.authproxy import JSONRPCException +from dataclasses import dataclass from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_greater_than_or_equal from threading import Thread +from typing import Optional import subprocess -def expect_http_status(expected_http_status, expected_rpc_code, - fcn, *args): - try: - fcn(*args) - raise AssertionError(f"Expected RPC error {expected_rpc_code}, got none") - except JSONRPCException as exc: - assert_equal(exc.error["code"], expected_rpc_code) - assert_equal(exc.http_status, expected_http_status) +RPC_INVALID_ADDRESS_OR_KEY = -5 +RPC_INVALID_PARAMETER = -8 +RPC_METHOD_NOT_FOUND = -32601 +RPC_INVALID_REQUEST = -32600 +RPC_PARSE_ERROR = -32700 + + +@dataclass +class BatchOptions: + version: Optional[int] = None + notification: bool = False + request_fields: Optional[dict] = None + response_fields: Optional[dict] = None + + +def format_request(options, idx, fields): + request = {} + if options.version == 1: + request.update(version="1.1") + elif options.version == 2: + request.update(jsonrpc="2.0") + elif options.version is not None: + raise NotImplementedError(f"Unknown JSONRPC version {options.version}") + if not options.notification: + request.update(id=idx) + request.update(fields) + if options.request_fields: + request.update(options.request_fields) + return request + + +def format_response(options, idx, fields): + if options.version == 2 and options.notification: + return None + response = {} + if not options.notification: + response.update(id=idx) + if options.version == 2: + response.update(jsonrpc="2.0") + else: + response.update(result=None, error=None) + response.update(fields) + if options.response_fields: + response.update(options.response_fields) + return response + + +def send_raw_rpc(node, raw_body: bytes) -> tuple[object, int]: + return node._request("POST", "/", raw_body) + + +def send_json_rpc(node, body: object) -> tuple[object, int]: + raw = json.dumps(body).encode("utf-8") + return send_raw_rpc(node, raw) + + +def expect_http_rpc_status(expected_http_status, expected_rpc_error_code, node, method, params, version=1, notification=False): + req = format_request(BatchOptions(version, notification), 0, {"method": method, "params": params}) + response, status = send_json_rpc(node, req) + + if expected_rpc_error_code is not None: + assert_equal(response["error"]["code"], expected_rpc_error_code) + + assert_equal(status, expected_http_status) def test_work_queue_getblock(node, got_exceeded_error): @@ -48,37 +106,126 @@ class RPCInterfaceTest(BitcoinTestFramework): assert_greater_than_or_equal(command['duration'], 0) assert_equal(info['logpath'], os.path.join(self.nodes[0].chain_path, 'debug.log')) - def test_batch_request(self): - self.log.info("Testing basic JSON-RPC batch request...") - - results = self.nodes[0].batch([ + def test_batch_request(self, call_options): + calls = [ # A basic request that will work fine. - {"method": "getblockcount", "id": 1}, + {"method": "getblockcount"}, # Request that will fail. The whole batch request should still # work fine. - {"method": "invalidmethod", "id": 2}, + {"method": "invalidmethod"}, # Another call that should succeed. - {"method": "getblockhash", "id": 3, "params": [0]}, - ]) - - result_by_id = {} - for res in results: - result_by_id[res["id"]] = res - - assert_equal(result_by_id[1]['error'], None) - assert_equal(result_by_id[1]['result'], 0) - - assert_equal(result_by_id[2]['error']['code'], -32601) - assert_equal(result_by_id[2]['result'], None) - - assert_equal(result_by_id[3]['error'], None) - assert result_by_id[3]['result'] is not None + {"method": "getblockhash", "params": [0]}, + # Invalid request format + {"pizza": "sausage"} + ] + results = [ + {"result": 0}, + {"error": {"code": RPC_METHOD_NOT_FOUND, "message": "Method not found"}}, + {"result": "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"}, + {"error": {"code": RPC_INVALID_REQUEST, "message": "Missing method"}}, + ] + + request = [] + response = [] + for idx, (call, result) in enumerate(zip(calls, results), 1): + options = call_options(idx) + if options is None: + continue + request.append(format_request(options, idx, call)) + r = format_response(options, idx, result) + if r is not None: + response.append(r) + + rpc_response, http_status = send_json_rpc(self.nodes[0], request) + if len(response) == 0 and len(request) > 0: + assert_equal(http_status, 204) + assert_equal(rpc_response, None) + else: + assert_equal(http_status, 200) + assert_equal(rpc_response, response) + + def test_batch_requests(self): + self.log.info("Testing empty batch request...") + self.test_batch_request(lambda idx: None) + + self.log.info("Testing basic JSON-RPC 2.0 batch request...") + self.test_batch_request(lambda idx: BatchOptions(version=2)) + + self.log.info("Testing JSON-RPC 2.0 batch with notifications...") + self.test_batch_request(lambda idx: BatchOptions(version=2, notification=idx < 2)) + + self.log.info("Testing JSON-RPC 2.0 batch of ALL notifications...") + self.test_batch_request(lambda idx: BatchOptions(version=2, notification=True)) + + # JSONRPC 1.1 does not support batch requests, but test them for backwards compatibility. + self.log.info("Testing nonstandard JSON-RPC 1.1 batch request...") + self.test_batch_request(lambda idx: BatchOptions(version=1)) + + self.log.info("Testing nonstandard mixed JSON-RPC 1.1/2.0 batch request...") + self.test_batch_request(lambda idx: BatchOptions(version=2 if idx % 2 else 1)) + + self.log.info("Testing nonstandard batch request without version numbers...") + self.test_batch_request(lambda idx: BatchOptions()) + + self.log.info("Testing nonstandard batch request without version numbers or ids...") + self.test_batch_request(lambda idx: BatchOptions(notification=True)) + + self.log.info("Testing nonstandard jsonrpc 1.0 version number is accepted...") + self.test_batch_request(lambda idx: BatchOptions(request_fields={"jsonrpc": "1.0"})) + + self.log.info("Testing unrecognized jsonrpc version number is rejected...") + self.test_batch_request(lambda idx: BatchOptions( + request_fields={"jsonrpc": "2.1"}, + response_fields={"result": None, "error": {"code": RPC_INVALID_REQUEST, "message": "JSON-RPC version not supported"}})) def test_http_status_codes(self): - self.log.info("Testing HTTP status codes for JSON-RPC requests...") - - expect_http_status(404, -32601, self.nodes[0].invalidmethod) - expect_http_status(500, -8, self.nodes[0].getblockhash, 42) + self.log.info("Testing HTTP status codes for JSON-RPC 1.1 requests...") + # OK + expect_http_rpc_status(200, None, self.nodes[0], "getblockhash", [0]) + # Errors + expect_http_rpc_status(404, RPC_METHOD_NOT_FOUND, self.nodes[0], "invalidmethod", []) + expect_http_rpc_status(500, RPC_INVALID_PARAMETER, self.nodes[0], "getblockhash", [42]) + # force-send empty request + response, status = send_raw_rpc(self.nodes[0], b"") + assert_equal(response, {"id": None, "result": None, "error": {"code": RPC_PARSE_ERROR, "message": "Parse error"}}) + assert_equal(status, 500) + # force-send invalidly formatted request + response, status = send_raw_rpc(self.nodes[0], b"this is bad") + assert_equal(response, {"id": None, "result": None, "error": {"code": RPC_PARSE_ERROR, "message": "Parse error"}}) + assert_equal(status, 500) + + self.log.info("Testing HTTP status codes for JSON-RPC 2.0 requests...") + # OK + expect_http_rpc_status(200, None, self.nodes[0], "getblockhash", [0], 2, False) + # RPC errors but not HTTP errors + expect_http_rpc_status(200, RPC_METHOD_NOT_FOUND, self.nodes[0], "invalidmethod", [], 2, False) + expect_http_rpc_status(200, RPC_INVALID_PARAMETER, self.nodes[0], "getblockhash", [42], 2, False) + # force-send invalidly formatted requests + response, status = send_json_rpc(self.nodes[0], {"jsonrpc": 2, "method": "getblockcount"}) + assert_equal(response, {"result": None, "error": {"code": RPC_INVALID_REQUEST, "message": "jsonrpc field must be a string"}}) + assert_equal(status, 400) + response, status = send_json_rpc(self.nodes[0], {"jsonrpc": "3.0", "method": "getblockcount"}) + assert_equal(response, {"result": None, "error": {"code": RPC_INVALID_REQUEST, "message": "JSON-RPC version not supported"}}) + assert_equal(status, 400) + + self.log.info("Testing HTTP status codes for JSON-RPC 2.0 notifications...") + # Not notification: id exists + response, status = send_json_rpc(self.nodes[0], {"jsonrpc": "2.0", "id": None, "method": "getblockcount"}) + assert_equal(response["result"], 0) + assert_equal(status, 200) + # Not notification: JSON 1.1 + expect_http_rpc_status(200, None, self.nodes[0], "getblockcount", [], 1) + # Not notification: has "id" field + expect_http_rpc_status(200, None, self.nodes[0], "getblockcount", [], 2, False) + block_count = self.nodes[0].getblockcount() + # Notification response status code: HTTP_NO_CONTENT + expect_http_rpc_status(204, None, self.nodes[0], "generatetoaddress", [1, "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqdku202"], 2, True) + # The command worked even though there was no response + assert_equal(block_count + 1, self.nodes[0].getblockcount()) + # No error response for notifications even if they are invalid + expect_http_rpc_status(204, None, self.nodes[0], "generatetoaddress", [1, "invalid_address"], 2, True) + # Sanity check: command was not executed + assert_equal(block_count + 1, self.nodes[0].getblockcount()) def test_work_queue_exceeded(self): self.log.info("Testing work queue exceeded...") @@ -94,7 +241,7 @@ class RPCInterfaceTest(BitcoinTestFramework): def run_test(self): self.test_getrpcinfo() - self.test_batch_request() + self.test_batch_requests() self.test_http_status_codes() self.test_work_queue_exceeded() diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 921c190668..98b397e32b 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -21,7 +21,6 @@ from test_framework.wallet import MiniWallet class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-maxorphantx=1000"]] def chain_tx(self, utxos_to_spend, *, num_outputs=1): return self.wallet.send_self_transfer_multi( diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index e83c62915e..4be6594de6 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -31,10 +31,8 @@ class MempoolPackagesTest(BitcoinTestFramework): self.noban_tx_relay = True self.extra_args = [ [ - "-maxorphantx=1000", ], [ - "-maxorphantx=1000", "-limitancestorcount={}".format(CUSTOM_ANCESTOR_LIMIT), "-limitdescendantcount={}".format(CUSTOM_DESCENDANT_LIMIT), ], diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 8ac0afdaaa..113424c0a6 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -23,6 +23,7 @@ from test_framework.util import ( assert_raises_rpc_error, ) from test_framework.wallet import ( + COIN, DEFAULT_FEE, MiniWallet, ) @@ -242,6 +243,37 @@ class RPCPackagesTest(BitcoinTestFramework): {"txid": tx2["txid"], "wtxid": tx2["wtxid"], "package-error": "conflict-in-package"} ]) + # Add a child that spends both at high feerate to submit via submitpackage + tx_child = self.wallet.create_self_transfer_multi( + fee_per_output=int(DEFAULT_FEE * 5 * COIN), + utxos_to_spend=[tx1["new_utxo"], tx2["new_utxo"]], + ) + + testres = node.testmempoolaccept([tx1["hex"], tx2["hex"], tx_child["hex"]]) + + assert_equal(testres, [ + {"txid": tx1["txid"], "wtxid": tx1["wtxid"], "package-error": "conflict-in-package"}, + {"txid": tx2["txid"], "wtxid": tx2["wtxid"], "package-error": "conflict-in-package"}, + {"txid": tx_child["txid"], "wtxid": tx_child["wtxid"], "package-error": "conflict-in-package"} + ]) + + submitres = node.submitpackage([tx1["hex"], tx2["hex"], tx_child["hex"]]) + assert_equal(submitres, {'package_msg': 'conflict-in-package', 'tx-results': {}, 'replaced-transactions': []}) + + # Submit tx1 to mempool, then try the same package again + node.sendrawtransaction(tx1["hex"]) + + submitres = node.submitpackage([tx1["hex"], tx2["hex"], tx_child["hex"]]) + assert_equal(submitres, {'package_msg': 'conflict-in-package', 'tx-results': {}, 'replaced-transactions': []}) + assert tx_child["txid"] not in node.getrawmempool() + + # ... and without the in-mempool ancestor tx1 included in the call + submitres = node.submitpackage([tx2["hex"], tx_child["hex"]]) + assert_equal(submitres, {'package_msg': 'package-not-child-with-unconfirmed-parents', 'tx-results': {}, 'replaced-transactions': []}) + + # Regardless of error type, the child can never enter the mempool + assert tx_child["txid"] not in node.getrawmempool() + def test_rbf(self): node = self.nodes[0] diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 03042877b2..7edf9f3679 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -160,6 +160,15 @@ class AuthServiceProxy(): raise JSONRPCException({ 'code': -342, 'message': 'missing HTTP response from server'}) + # Check for no-content HTTP status code, which can be returned when an + # RPC client requests a JSON-RPC 2.0 "notification" with no response. + # Currently this is only possible if clients call the _request() method + # directly to send a raw request. + if http_response.status == HTTPStatus.NO_CONTENT: + if len(http_response.read()) != 0: + raise JSONRPCException({'code': -342, 'message': 'Content received with NO CONTENT status code'}) + return None, http_response.status + content_type = http_response.getheader('Content-Type') if content_type != 'application/json': raise JSONRPCException( diff --git a/test/functional/test_framework/crypto/secp256k1.py b/test/functional/test_framework/crypto/secp256k1.py index 2e9e419da5..50a46dce37 100644 --- a/test/functional/test_framework/crypto/secp256k1.py +++ b/test/functional/test_framework/crypto/secp256k1.py @@ -15,6 +15,8 @@ Exports: * G: the secp256k1 generator point """ +import unittest +from hashlib import sha256 class FE: """Objects of this class represent elements of the field GF(2**256 - 2**32 - 977). @@ -344,3 +346,9 @@ class FastGEMul: # Precomputed table with multiples of G for fast multiplication FAST_G = FastGEMul(G) + +class TestFrameworkSecp256k1(unittest.TestCase): + def test_H(self): + H = sha256(G.to_bytes_uncompressed()).digest() + assert GE.lift_x(FE.from_bytes(H)) is not None + self.assertEqual(H.hex(), "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d228bd8991..4ba92a7b1f 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -419,8 +419,9 @@ class TestNode(): return True def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs): - expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS - self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout) + if "expected_ret_code" not in kwargs: + kwargs["expected_ret_code"] = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS + self.wait_until(lambda: self.is_node_stopped(**kwargs), timeout=timeout) def replace_in_config(self, replacements): """ diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 690ab64c83..725b116281 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -183,6 +183,8 @@ BASE_SCRIPTS = [ 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', + 'tool_wallet.py --legacy-wallet --bdbro', + 'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian', 'tool_wallet.py --descriptors', 'tool_signet_miner.py --legacy-wallet', 'tool_signet_miner.py --descriptors', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index fc042bca66..dcf74f6075 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -5,6 +5,7 @@ """Test bitcoin-wallet.""" import os +import platform import stat import subprocess import textwrap @@ -14,6 +15,7 @@ from collections import OrderedDict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, sha256sum_file, ) @@ -21,11 +23,15 @@ from test_framework.util import ( class ToolWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) + parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file") + parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally") def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.rpc_timeout = 120 + if self.options.swap_bdb_endian: + self.extra_args = [["-swapbdbendian"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -35,15 +41,21 @@ class ToolWalletTest(BitcoinTestFramework): default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain] if not self.options.descriptors and 'create' in args: default_args.append('-legacy') + if "dump" in args and self.options.bdbro: + default_args.append("-withinternalbdb") return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) stdout, stderr = p.communicate() - assert_equal(p.poll(), 1) assert_equal(stdout, '') - assert_equal(stderr.strip(), error) + if isinstance(error, tuple): + assert_equal(p.poll(), error[0]) + assert error[1] in stderr.strip() + else: + assert_equal(p.poll(), 1) + assert error in stderr.strip() def assert_tool_output(self, output, *args): p = self.bitcoin_wallet_process(*args) @@ -451,6 +463,88 @@ class ToolWalletTest(BitcoinTestFramework): ''') self.assert_tool_output(expected_output, "-wallet=conflicts", "info") + def test_dump_endianness(self): + self.log.info("Testing dumps of the same contents with different BDB endianness") + + self.start_node(0) + self.nodes[0].createwallet("endian") + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "endian.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=endian", f"-dumpfile={wallet_dump}", "dump") + expected_dump = self.read_dump(wallet_dump) + + self.do_tool_createfromdump("native_endian", "endian.dump", "bdb") + native_dump = self.read_dump(self.nodes[0].datadir_path / "rt-native_endian.dump") + self.assert_dump(expected_dump, native_dump) + + self.do_tool_createfromdump("other_endian", "endian.dump", "bdb_swap") + other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump") + self.assert_dump(expected_dump, other_dump) + + def test_dump_very_large_records(self): + self.log.info("Test that wallets with large records are successfully dumped") + + self.start_node(0) + self.nodes[0].createwallet("bigrecords") + wallet = self.nodes[0].get_wallet_rpc("bigrecords") + + # Both BDB and sqlite have maximum page sizes of 65536 bytes, with defaults of 4096 + # When a record exceeds some size threshold, both BDB and SQLite will store the data + # in one or more overflow pages. We want to make sure that our tooling can dump such + # records, even when they span multiple pages. To make a large record, we just need + # to make a very big transaction. + self.generate(self.nodes[0], 101) + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + outputs = {} + for i in range(500): + outputs[wallet.getnewaddress(address_type="p2sh-segwit")] = 0.01 + def_wallet.sendmany(amounts=outputs) + self.generate(self.nodes[0], 1) + send_res = wallet.sendall([def_wallet.getnewaddress()]) + self.generate(self.nodes[0], 1) + assert_equal(send_res["complete"], True) + tx = wallet.gettransaction(txid=send_res["txid"], verbose=True) + assert_greater_than(tx["decoded"]["size"], 70000) + + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "bigrecords.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=bigrecords", f"-dumpfile={wallet_dump}", "dump") + dump = self.read_dump(wallet_dump) + for k,v in dump.items(): + if tx["hex"] in v: + break + else: + assert False, "Big transaction was not found in wallet dump" + + def test_dump_unclean_lsns(self): + if not self.options.bdbro: + return + self.log.info("Test that a legacy wallet that has not been compacted is not dumped by bdbro") + + self.start_node(0, extra_args=["-flushwallet=0"]) + self.nodes[0].createwallet("unclean_lsn") + wallet = self.nodes[0].get_wallet_rpc("unclean_lsn") + # First unload and load normally to make sure everything is written + wallet.unloadwallet() + self.nodes[0].loadwallet("unclean_lsn") + # Next cause a bunch of writes by filling the keypool + wallet.keypoolrefill(wallet.getwalletinfo()["keypoolsize"] + 100) + # Lastly kill bitcoind so that the LSNs don't get reset + self.nodes[0].process.kill() + self.nodes[0].wait_until_stopped(expected_ret_code=1 if platform.system() == "Windows" else -9) + assert self.nodes[0].is_node_stopped() + + wallet_dump = self.nodes[0].datadir_path / "unclean_lsn.dump" + self.assert_raises_tool_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + + # File can be dumped after reload it normally + self.start_node(0) + self.nodes[0].loadwallet("unclean_lsn") + self.stop_node(0) + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -462,8 +556,11 @@ class ToolWalletTest(BitcoinTestFramework): if not self.options.descriptors: # Salvage is a legacy wallet only thing self.test_salvage() + self.test_dump_endianness() + self.test_dump_unclean_lsns() self.test_dump_createfromdump() self.test_chainless_conflicts() + self.test_dump_very_large_records() if __name__ == '__main__': ToolWalletTest().main() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index a635175e7c..c74246ef45 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -215,12 +215,12 @@ def transform_process_message_target(targets, src_dir): p2p_msg_target = "process_message" if (p2p_msg_target, {}) in targets: lines = subprocess.run( - ["git", "grep", "--function-context", "g_all_net_message_types{", src_dir / "src" / "protocol.cpp"], + ["git", "grep", "--function-context", "ALL_NET_MESSAGE_TYPES{", src_dir / "src" / "protocol.h"], check=True, stdout=subprocess.PIPE, text=True, ).stdout.splitlines() - lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.cpp- NetMsgType::")] + lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.h- NetMsgType::")] assert len(lines) targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines] return targets |