diff options
67 files changed, 1848 insertions, 612 deletions
diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index 48dfafaee0..c2c32af838 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -10,6 +10,9 @@ </PropertyGroup> <ItemGroup> <ClCompile Include="..\..\src\bitcoind.cpp" /> + <ClCompile Include="..\..\src\init\bitcoind.cpp"> + <ObjectFileName>$(IntDir)init_bitcoind.obj</ObjectFileName> + </ClCompile> </ItemGroup> <ItemGroup> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 490ce8b1ce..96bb584375 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -34,6 +34,7 @@ <ClCompile Include="..\..\src\qt\overviewpage.cpp" /> <ClCompile Include="..\..\src\qt\paymentserver.cpp" /> <ClCompile Include="..\..\src\qt\peertablemodel.cpp" /> + <ClCompile Include="..\..\src\qt\peertablesortproxy.cpp" /> <ClCompile Include="..\..\src\qt\platformstyle.cpp" /> <ClCompile Include="..\..\src\qt\psbtoperationsdialog.cpp" /> <ClCompile Include="..\..\src\qt\qrimagewidget.cpp" /> @@ -87,6 +88,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_overviewpage.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_paymentserver.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablemodel.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablesortproxy.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_platformstyle.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_psbtoperationsdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qrimagewidget.cpp" /> diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index c50fdc3f6b..14653ce9fb 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0 +$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4 +$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370 $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 5fe027fb8a..783f87ca7c 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,8 +1,8 @@ package=native_mac_alias -$(package)_version=2.1.1 +$(package)_version=2.2.0 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=c0ffceee14f7d04a6eb323fb7b8217dc3f373b346198d2ca42300a8362db7efa +$(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 7a42fdd734..e3f389a6d3 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -15,7 +15,7 @@ Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102 ## Debugging -After [#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes. +The `-debug=ipc` command line option can be used to see requests and responses between processes. ## Installation @@ -33,3 +33,40 @@ BITCOIND=bitcoin-node test/functional/test_runner.py The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. + +## IPC implementation details + +Cross process Node, Wallet, and Chain interfaces are defined in +[`src/interfaces/`](../src/interfaces/). These are C++ classes which follow +[conventions](developer-notes.md#internal-interface-guidelines), like passing +serializable arguments so they can be called from different processes, and +making methods pure virtual so they can have proxy implementations that forward +calls between processes. + +When Wallet, Node, and Chain code is running in the same process, calling any +interface method invokes the implementation directly. When code is running in +different processes, calling an interface method invokes a proxy interface +implementation that communicates with a remote process and invokes the real +implementation in the remote process. The +[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) code +generation tool internally generates proxy client classes and proxy server +classes for this purpose that are thin wrappers around Cap'n Proto +[client](https://capnproto.org/cxxrpc.html#clients) and +[server](https://capnproto.org/cxxrpc.html#servers) classes, which handle the +actual serialization and socket communication. + +As much as possible, calls between processes are meant to work the same as +calls within a single process without adding limitations or requiring extra +implementation effort. Processes communicate with each other by calling regular +[C++ interface methods](../src/interfaces/README.md). Method arguments and +return values are automatically serialized and sent between processes. Object +references and `std::function` arguments are automatically tracked and mapped +to allow invoked code to call back into invoking code at any time, and there is +a 1:1 threading model where any thread invoking a method in another process has +a corresponding thread in the invoked process responsible for executing all +method calls from the source thread, without blocking I/O or holding up another +call, and using the same thread local variables, locks, and callbacks between +calls. The forwarding, tracking, and threading is implemented inside the +[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library +which has the design goal of making calls between processes look like calls in +the same process to the extent possible. diff --git a/src/Makefile.am b/src/Makefile.am index e25e210e1e..447015fc66 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -74,6 +74,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_CLI) \ + $(LIBBITCOIN_IPC) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_ZMQ) @@ -156,8 +157,12 @@ BITCOIN_CORE_H = \ index/txindex.h \ indirectmap.h \ init.h \ + init/common.h \ interfaces/chain.h \ + interfaces/echo.h \ interfaces/handler.h \ + interfaces/init.h \ + interfaces/ipc.h \ interfaces/node.h \ interfaces/wallet.h \ key.h \ @@ -298,6 +303,8 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h +ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) + # server: shared between bitcoind and bitcoin-qt # Contains code accessing mempool and chain state that is meant to be separated # from wallet and gui code (see node/README.md). Shared code should go in @@ -521,6 +528,7 @@ libbitcoin_common_a_SOURCES = \ core_read.cpp \ core_write.cpp \ external_signer.cpp \ + init/common.cpp \ key.cpp \ key_io.cpp \ merkleblock.cpp \ @@ -556,7 +564,9 @@ libbitcoin_util_a_SOURCES = \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ fs.cpp \ + interfaces/echo.cpp \ interfaces/handler.cpp \ + interfaces/init.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ @@ -632,17 +642,17 @@ bitcoin_bin_ldadd = \ bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) -bitcoind_SOURCES = $(bitcoin_daemon_sources) +bitcoind_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoind_LDFLAGS = $(bitcoin_bin_ldflags) bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) -bitcoin_node_SOURCES = $(bitcoin_daemon_sources) +bitcoin_node_SOURCES = $(bitcoin_daemon_sources) init/bitcoin-node.cpp bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) +bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -806,6 +816,39 @@ if HARDEN $(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif +libbitcoin_ipc_mpgen_input = \ + ipc/capnp/echo.capnp \ + ipc/capnp/init.capnp +EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) +%.capnp: + +if BUILD_MULTIPROCESS +LIBBITCOIN_IPC=libbitcoin_ipc.a +libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/init-types.h \ + ipc/capnp/protocol.cpp \ + ipc/capnp/protocol.h \ + ipc/exception.h \ + ipc/interfaces.cpp \ + ipc/process.cpp \ + ipc/process.h \ + ipc/protocol.h +libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_ipc_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS) + +include $(MPGEN_PREFIX)/include/mpgen.mk +libbitcoin_ipc_mpgen_output = \ + $(libbitcoin_ipc_mpgen_input:=.c++) \ + $(libbitcoin_ipc_mpgen_input:=.h) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-client.c++) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-server.c++) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-types.c++) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-types.h) \ + $(libbitcoin_ipc_mpgen_input:=.proxy.h) +nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) +CLEANFILES += $(libbitcoin_ipc_mpgen_output) +endif + if EMBEDDED_LEVELDB include Makefile.crc32c.include include Makefile.leveldb.include diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index a573247afa..caa8500ffa 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -61,6 +61,7 @@ QT_MOC_CPP = \ qt/moc_optionsmodel.cpp \ qt/moc_overviewpage.cpp \ qt/moc_peertablemodel.cpp \ + qt/moc_peertablesortproxy.cpp \ qt/moc_paymentserver.cpp \ qt/moc_psbtoperationsdialog.cpp \ qt/moc_qrimagewidget.cpp \ @@ -134,6 +135,7 @@ BITCOIN_QT_H = \ qt/overviewpage.h \ qt/paymentserver.h \ qt/peertablemodel.h \ + qt/peertablesortproxy.h \ qt/platformstyle.h \ qt/psbtoperationsdialog.h \ qt/qrimagewidget.h \ @@ -232,6 +234,7 @@ BITCOIN_QT_BASE_CPP = \ qt/optionsdialog.cpp \ qt/optionsmodel.cpp \ qt/peertablemodel.cpp \ + qt/peertablesortproxy.cpp \ qt/platformstyle.cpp \ qt/qvalidatedlineedit.cpp \ qt/qvaluecombobox.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 570f011f7a..efddc5e8c4 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -274,6 +274,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/random.cpp \ test/fuzz/rbf.cpp \ test/fuzz/rolling_bloom_filter.cpp \ + test/fuzz/rpc.cpp \ test/fuzz/script.cpp \ test/fuzz/script_assets_test_minimizer.cpp \ test/fuzz/script_bitcoin_consensus.cpp \ diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 225b8b1ec4..cf9e4fad44 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -12,6 +12,7 @@ #include <compat.h> #include <init.h> #include <interfaces/chain.h> +#include <interfaces/init.h> #include <node/context.h> #include <node/ui_interface.h> #include <noui.h> @@ -104,10 +105,8 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint) #endif -static bool AppInit(int argc, char* argv[]) +static bool AppInit(NodeContext& node, int argc, char* argv[]) { - NodeContext node; - bool fRet = false; util::ThreadSetInternalName("init"); @@ -254,10 +253,18 @@ int main(int argc, char* argv[]) util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); #endif + + NodeContext node; + int exit_status; + std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node, argc, argv, exit_status); + if (!init) { + return exit_status; + } + SetupEnvironment(); // Connect bitcoind signal handlers noui_connect(); - return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); + return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/src/compressor.cpp b/src/compressor.cpp index a70306d320..ef3135e7a5 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -52,7 +52,7 @@ static bool IsToPubKey(const CScript& script, CPubKey &pubkey) return false; } -bool CompressScript(const CScript& script, std::vector<unsigned char> &out) +bool CompressScript(const CScript& script, CompressedScript& out) { CKeyID keyID; if (IsToKeyID(script, keyID)) { @@ -92,7 +92,7 @@ unsigned int GetSpecialScriptSize(unsigned int nSize) return 0; } -bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &in) +bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScript& in) { switch(nSize) { case 0x00: diff --git a/src/compressor.h b/src/compressor.h index 478bfff0b6..40b2496f06 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -6,14 +6,26 @@ #ifndef BITCOIN_COMPRESSOR_H #define BITCOIN_COMPRESSOR_H +#include <prevector.h> #include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> #include <span.h> -bool CompressScript(const CScript& script, std::vector<unsigned char> &out); +/** + * This saves us from making many heap allocations when serializing + * and deserializing compressed scripts. + * + * This prevector size is determined by the largest .resize() in the + * CompressScript function. The largest compressed script format is a + * compressed public key, which is 33 bytes. + */ +using CompressedScript = prevector<33, unsigned char>; + + +bool CompressScript(const CScript& script, CompressedScript& out); unsigned int GetSpecialScriptSize(unsigned int nSize); -bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); +bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScript& in); /** * Compress amount. @@ -51,7 +63,7 @@ struct ScriptCompression template<typename Stream> void Ser(Stream &s, const CScript& script) { - std::vector<unsigned char> compr; + CompressedScript compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); return; @@ -66,7 +78,7 @@ struct ScriptCompression unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { - std::vector<unsigned char> vch(GetSpecialScriptSize(nSize), 0x00); + CompressedScript vch(GetSpecialScriptSize(nSize), 0x00); s >> MakeSpan(vch); DecompressScript(script, nSize, vch); return; diff --git a/src/init.cpp b/src/init.cpp index 741e70f748..3f9838f2e0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -22,9 +22,9 @@ #include <httpserver.h> #include <index/blockfilterindex.h> #include <index/txindex.h> +#include <init/common.h> #include <interfaces/chain.h> #include <interfaces/node.h> -#include <key.h> #include <mapport.h> #include <miner.h> #include <net.h> @@ -151,8 +151,6 @@ static fs::path GetPidFile(const ArgsManager& args) // shutdown thing. // -static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; - void Interrupt(NodeContext& node) { InterruptHTTPServer(); @@ -196,20 +194,7 @@ void Shutdown(NodeContext& node) // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. if (node.peerman) UnregisterValidationInterface(node.peerman.get()); - // Follow the lock order requirements: - // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraFullOutboundCount - // which locks cs_vNodes. - // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which - // locks cs_vNodes. - // * CConnman::Stop calls DeleteNode, which calls FinalizeNode, which locks cs_main and calls - // EraseOrphansFor, which locks g_cs_orphans. - // - // Thus the implicit locking order requirement is: (1) cs_main, (2) g_cs_orphans, (3) cs_vNodes. - if (node.connman) { - node.connman->StopThreads(); - LOCK2(::cs_main, ::g_cs_orphans); - node.connman->StopNodes(); - } + if (node.connman) node.connman->Stop(); StopTorControl(); @@ -286,8 +271,7 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - globalVerifyHandle.reset(); - ECC_Stop(); + init::UnsetGlobals(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman = nullptr; @@ -363,6 +347,8 @@ void SetupServerArgs(NodeContext& node) SetupHelpOptions(argsman); argsman.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now + init::AddLoggingArgs(argsman); + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); const auto signetBaseParams = CreateBaseChainParams(CBaseChainParams::SIGNET); @@ -394,7 +380,6 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -522,25 +507,10 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", - ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); -#ifdef HAVE_THREAD_LOCAL - argsman.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); -#else - hidden_args.emplace_back("-logthreadnames"); -#endif - argsman.AddArg("-logsourcelocations", strprintf("Prepend debug output with name of the originating source location (source file, line number and function name) (default: %u)", DEFAULT_LOGSOURCELOCATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); SetupChainParamsBaseOptions(argsman); @@ -675,30 +645,6 @@ static void StartupNotify(const ArgsManager& args) } #endif -/** Sanity checks - * Ensure that Bitcoin is running in a usable environment with all - * necessary library support. - */ -static bool InitSanityCheck() -{ - if (!ECC_InitSanityCheck()) { - return InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); - } - - if (!glibcxx_sanity_test()) - return false; - - if (!Random_SanityCheck()) { - return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); - } - - if (!ChronoSanityCheck()) { - return InitError(Untranslated("Clock epoch mismatch. Aborting.")); - } - - return true; -} - static bool AppInitServers(NodeContext& node) { const ArgsManager& args = *Assert(node.args); @@ -796,25 +742,8 @@ void InitParameterInteraction(ArgsManager& args) */ void InitLogging(const ArgsManager& args) { - LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); - LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); - LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); - LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); -#ifdef HAVE_THREAD_LOCAL - LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); -#endif - LogInstance().m_log_sourcelocations = args.GetBoolArg("-logsourcelocations", DEFAULT_LOGSOURCELOCATIONS); - - fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); - - std::string version_string = FormatFullVersion(); -#ifdef DEBUG - version_string += " (debug build)"; -#else - version_string += " (release build)"; -#endif - LogPrintf(PACKAGE_NAME " version %s\n", version_string); + init::SetLoggingOptions(args); + init::LogPackageVersion(); } namespace { // Variables internal to initialization process only @@ -982,26 +911,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (args.IsArgSet("-debug")) { - // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string> categories = args.GetArgs("-debug"); - - if (std::none_of(categories.begin(), categories.end(), - [](std::string cat){return cat == "0" || cat == "none";})) { - for (const auto& cat : categories) { - if (!LogInstance().EnableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); - } - } - } - } - - // Now remove the logging categories which were explicitly excluded - for (const std::string& cat : args.GetArgs("-debugexclude")) { - if (!LogInstance().DisableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); - } - } + init::SetLoggingCategories(args); fCheckBlockIndex = args.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); fCheckpointsEnabled = args.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); @@ -1148,16 +1058,11 @@ bool AppInitSanityChecks() { // ********************************************************* Step 4: sanity checks - // Initialize elliptic curve code - std::string sha256_algo = SHA256AutoDetect(); - LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); - RandomInit(); - ECC_Start(); - globalVerifyHandle.reset(new ECCVerifyHandle()); + init::SetGlobals(); - // Sanity check - if (!InitSanityCheck()) + if (!init::SanityChecks()) { return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); + } // Probe the data directory lock to give an early error message, if possible // We cannot hold the data directory lock here, as the forking for daemon() hasn't yet happened, @@ -1197,38 +1102,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Detailed error printed inside CreatePidFile(). return false; } - if (LogInstance().m_print_to_file) { - if (args.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { - // Do this first since it both loads a bunch of debug.log into memory, - // and because this needs to happen before any other debug.log printing - LogInstance().ShrinkDebugFile(); - } - } - if (!LogInstance().StartLogging()) { - return InitError(strprintf(Untranslated("Could not open debug log file %s"), - LogInstance().m_file_path.string())); - } - - if (!LogInstance().m_log_timestamps) - LogPrintf("Startup time: %s\n", FormatISO8601DateTime(GetTime())); - LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); - LogPrintf("Using data directory %s\n", GetDataDir().string()); - - // Only log conf file usage message if conf file actually exists. - fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); - if (fs::exists(config_file_path)) { - LogPrintf("Config file: %s\n", config_file_path.string()); - } else if (args.IsArgSet("-conf")) { - // Warn if no conf file exists at path provided by user - InitWarning(strprintf(_("The specified config file %s does not exist"), config_file_path.string())); - } else { - // Not categorizing as "Warning" because it's the default behavior - LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); + if (!init::StartLogging(args)) { + // Detailed error printed inside StartLogging(). + return false; } - // Log the config arguments to debug.log - args.LogArgs(); - LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. @@ -1492,7 +1370,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { - bool fReset = fReindex; + const bool fReset = fReindex; auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); }; @@ -1613,29 +1491,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) break; } - bool failed_rewind{false}; - // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant - // chainstates beforehand. - for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on the chainstate, and drops block data in - // BlockIndex() based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...").translated); - if (!chainstate->RewindBlockIndex(chainparams)) { - strLoadError = _( - "Unable to rewind the database to a pre-fork state. " - "You will need to redownload the blockchain"); - failed_rewind = true; - break; // out of the per-chainstate loop - } + if (!fReset) { + LOCK(cs_main); + auto chainstates{chainman.GetAll()}; + if (std::any_of(chainstates.begin(), chainstates.end(), + [&chainparams](const CChainState* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(chainparams); })) { + strLoadError = strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), + chainparams.GetConsensus().SegwitHeight); + break; } } - if (failed_rewind) { - break; // out of the chainstate activation do-while - } - bool failed_verification = false; try { @@ -1659,11 +1525,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) break; } - // Only verify the DB of the active chainstate. This is fixed in later - // work when we allow VerifyDB to be parameterized by chainstate. - if (&::ChainstateActive() == chainstate && - !CVerifyDB().VerifyDB( - chainparams, *chainstate, &chainstate->CoinsDB(), + if (!CVerifyDB().VerifyDB( + *chainstate, chainparams, chainstate->CoinsDB(), args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp new file mode 100644 index 0000000000..49684ede83 --- /dev/null +++ b/src/init/bitcoin-node.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> +#include <node/context.h> + +#include <memory> + +namespace init { +namespace { +const char* EXE_NAME = "bitcoin-node"; + +class BitcoinNodeInit : public interfaces::Init +{ +public: + BitcoinNodeInit(NodeContext& node, const char* arg0) + : m_node(node), + m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) + { + m_node.init = this; + } + std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } + interfaces::Ipc* ipc() override { return m_ipc.get(); } + NodeContext& m_node; + std::unique_ptr<interfaces::Ipc> m_ipc; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status) +{ + auto init = std::make_unique<init::BitcoinNodeInit>(node, argc > 0 ? argv[0] : ""); + // Check if bitcoin-node is being invoked as an IPC server. If so, then + // bypass normal execution and just respond to requests over the IPC + // channel and return null. + if (init->m_ipc->startSpawnedProcess(argc, argv, exit_status)) { + return nullptr; + } + return init; +} +} // namespace interfaces diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp new file mode 100644 index 0000000000..1e17ce4d3c --- /dev/null +++ b/src/init/bitcoind.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/init.h> +#include <node/context.h> + +#include <memory> + +namespace init { +namespace { +class BitcoindInit : public interfaces::Init +{ +public: + BitcoindInit(NodeContext& node) : m_node(node) + { + m_node.init = this; + } + NodeContext& m_node; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status) +{ + return std::make_unique<init::BitcoindInit>(node); +} +} // namespace interfaces diff --git a/src/init/common.cpp b/src/init/common.cpp new file mode 100644 index 0000000000..79e0c9da78 --- /dev/null +++ b/src/init/common.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <clientversion.h> +#include <compat/sanity.h> +#include <crypto/sha256.h> +#include <key.h> +#include <logging.h> +#include <node/ui_interface.h> +#include <pubkey.h> +#include <random.h> +#include <util/system.h> +#include <util/time.h> +#include <util/translation.h> + +#include <memory> + +static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; + +namespace init { +void SetGlobals() +{ + std::string sha256_algo = SHA256AutoDetect(); + LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); + RandomInit(); + ECC_Start(); + globalVerifyHandle.reset(new ECCVerifyHandle()); +} + +void UnsetGlobals() +{ + globalVerifyHandle.reset(); + ECC_Stop(); +} + +bool SanityChecks() +{ + if (!ECC_InitSanityCheck()) { + return InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); + } + + if (!glibcxx_sanity_test()) + return false; + + if (!Random_SanityCheck()) { + return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); + } + + if (!ChronoSanityCheck()) { + return InitError(Untranslated("Clock epoch mismatch. Aborting.")); + } + + return true; +} + +void AddLoggingArgs(ArgsManager& argsman) +{ + argsman.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", + ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#ifdef HAVE_THREAD_LOCAL + argsman.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#else + argsman.AddHiddenArgs({"-logthreadnames"}); +#endif + argsman.AddArg("-logsourcelocations", strprintf("Prepend debug output with name of the originating source location (source file, line number and function name) (default: %u)", DEFAULT_LOGSOURCELOCATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +} + +void SetLoggingOptions(const ArgsManager& args) +{ + LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); + LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); + LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); + LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); +#ifdef HAVE_THREAD_LOCAL + LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); +#endif + LogInstance().m_log_sourcelocations = args.GetBoolArg("-logsourcelocations", DEFAULT_LOGSOURCELOCATIONS); + + fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); +} + +void SetLoggingCategories(const ArgsManager& args) +{ + if (args.IsArgSet("-debug")) { + // Special-case: if -debug=0/-nodebug is set, turn off debugging messages + const std::vector<std::string> categories = args.GetArgs("-debug"); + + if (std::none_of(categories.begin(), categories.end(), + [](std::string cat){return cat == "0" || cat == "none";})) { + for (const auto& cat : categories) { + if (!LogInstance().EnableCategory(cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); + } + } + } + } + + // Now remove the logging categories which were explicitly excluded + for (const std::string& cat : args.GetArgs("-debugexclude")) { + if (!LogInstance().DisableCategory(cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); + } + } +} + +bool StartLogging(const ArgsManager& args) +{ + if (LogInstance().m_print_to_file) { + if (args.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { + // Do this first since it both loads a bunch of debug.log into memory, + // and because this needs to happen before any other debug.log printing + LogInstance().ShrinkDebugFile(); + } + } + if (!LogInstance().StartLogging()) { + return InitError(strprintf(Untranslated("Could not open debug log file %s"), + LogInstance().m_file_path.string())); + } + + if (!LogInstance().m_log_timestamps) + LogPrintf("Startup time: %s\n", FormatISO8601DateTime(GetTime())); + LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); + LogPrintf("Using data directory %s\n", GetDataDir().string()); + + // Only log conf file usage message if conf file actually exists. + fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); + if (fs::exists(config_file_path)) { + LogPrintf("Config file: %s\n", config_file_path.string()); + } else if (args.IsArgSet("-conf")) { + // Warn if no conf file exists at path provided by user + InitWarning(strprintf(_("The specified config file %s does not exist"), config_file_path.string())); + } else { + // Not categorizing as "Warning" because it's the default behavior + LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); + } + + // Log the config arguments to debug.log + args.LogArgs(); + + return true; +} + +void LogPackageVersion() +{ + std::string version_string = FormatFullVersion(); +#ifdef DEBUG + version_string += " (debug build)"; +#else + version_string += " (release build)"; +#endif + LogPrintf(PACKAGE_NAME " version %s\n", version_string); +} +} // namespace init diff --git a/src/init/common.h b/src/init/common.h new file mode 100644 index 0000000000..fc4bc1b280 --- /dev/null +++ b/src/init/common.h @@ -0,0 +1,28 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file +//! @brief Common init functions shared by bitcoin-node, bitcoin-wallet, etc. + +#ifndef BITCOIN_INIT_COMMON_H +#define BITCOIN_INIT_COMMON_H + +class ArgsManager; + +namespace init { +void SetGlobals(); +void UnsetGlobals(); +/** + * Ensure a usable environment with all + * necessary library support. + */ +bool SanityChecks(); +void AddLoggingArgs(ArgsManager& args); +void SetLoggingOptions(const ArgsManager& args); +void SetLoggingCategories(const ArgsManager& args); +bool StartLogging(const ArgsManager& args); +void LogPackageVersion(); +} // namespace init + +#endif // BITCOIN_INIT_COMMON_H diff --git a/src/interfaces/README.md b/src/interfaces/README.md index f77d172153..97167d5298 100644 --- a/src/interfaces/README.md +++ b/src/interfaces/README.md @@ -12,6 +12,8 @@ The following interfaces are defined here: * [`Handler`](handler.h) — returned by `handleEvent` methods on interfaces above and used to manage lifetimes of event handlers. -* [`Init`](init.h) — used by multiprocess code to access interfaces above on startup. Added in [#10102](https://github.com/bitcoin/bitcoin/pull/10102). +* [`Init`](init.h) — used by multiprocess code to access interfaces above on startup. Added in [#19160](https://github.com/bitcoin/bitcoin/pull/19160). -The interfaces above define boundaries between major components of bitcoin code (node, wallet, and gui), making it possible for them to run in different processes, and be tested, developed, and understood independently. These interfaces are not currently designed to be stable or to be used externally. +* [`Ipc`](ipc.h) — used by multiprocess code to access `Init` interface across processes. Added in [#19160](https://github.com/bitcoin/bitcoin/pull/19160). + +The interfaces above define boundaries between major components of bitcoin code (node, wallet, and gui), making it possible for them to run in [different processes](../../doc/multiprocess.md), and be tested, developed, and understood independently. These interfaces are not currently designed to be stable or to be used externally. diff --git a/src/interfaces/echo.cpp b/src/interfaces/echo.cpp new file mode 100644 index 0000000000..9bbb42217b --- /dev/null +++ b/src/interfaces/echo.cpp @@ -0,0 +1,18 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/echo.h> + +#include <memory> + +namespace interfaces { +namespace { +class EchoImpl : public Echo +{ +public: + std::string echo(const std::string& echo) override { return echo; } +}; +} // namespace +std::unique_ptr<Echo> MakeEcho() { return std::make_unique<EchoImpl>(); } +} // namespace interfaces diff --git a/src/interfaces/echo.h b/src/interfaces/echo.h new file mode 100644 index 0000000000..5578d9d9e6 --- /dev/null +++ b/src/interfaces/echo.h @@ -0,0 +1,26 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_ECHO_H +#define BITCOIN_INTERFACES_ECHO_H + +#include <memory> +#include <string> + +namespace interfaces { +//! Simple string echoing interface for testing. +class Echo +{ +public: + virtual ~Echo() {} + + //! Echo provided string. + virtual std::string echo(const std::string& echo) = 0; +}; + +//! Return implementation of Echo interface. +std::unique_ptr<Echo> MakeEcho(); +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_ECHO_H diff --git a/src/interfaces/init.cpp b/src/interfaces/init.cpp new file mode 100644 index 0000000000..a3c949e616 --- /dev/null +++ b/src/interfaces/init.cpp @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> + +namespace interfaces { +std::unique_ptr<Node> Init::makeNode() { return {}; } +std::unique_ptr<Chain> Init::makeChain() { return {}; } +std::unique_ptr<WalletClient> Init::makeWalletClient(Chain& chain) { return {}; } +std::unique_ptr<Echo> Init::makeEcho() { return {}; } +Ipc* Init::ipc() { return nullptr; } +} // namespace interfaces diff --git a/src/interfaces/init.h b/src/interfaces/init.h new file mode 100644 index 0000000000..2a38054a17 --- /dev/null +++ b/src/interfaces/init.h @@ -0,0 +1,52 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_INIT_H +#define BITCOIN_INTERFACES_INIT_H + +#include <memory> + +struct NodeContext; + +namespace interfaces { +class Chain; +class Echo; +class Ipc; +class Node; +class WalletClient; + +//! Initial interface created when a process is first started, and used to give +//! and get access to other interfaces (Node, Chain, Wallet, etc). +//! +//! There is a different Init interface implementation for each process +//! (bitcoin-gui, bitcoin-node, bitcoin-wallet, bitcoind, bitcoin-qt) and each +//! implementation can implement the make methods for interfaces it supports. +//! The default make methods all return null. +class Init +{ +public: + virtual ~Init() = default; + virtual std::unique_ptr<Node> makeNode(); + virtual std::unique_ptr<Chain> makeChain(); + virtual std::unique_ptr<WalletClient> makeWalletClient(Chain& chain); + virtual std::unique_ptr<Echo> makeEcho(); + virtual Ipc* ipc(); +}; + +//! Return implementation of Init interface for the node process. If the argv +//! indicates that this is a child process spawned to handle requests from a +//! parent process, this blocks and handles requests, then returns null and a +//! status code to exit with. If this returns non-null, the caller can start up +//! normally and use the Init object to spawn and connect to other processes +//! while it is running. +std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status); + +//! Return implementation of Init interface for the wallet process. +std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status); + +//! Return implementation of Init interface for the gui process. +std::unique_ptr<Init> MakeGuiInit(int argc, char* argv[]); +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_INIT_H diff --git a/src/interfaces/ipc.h b/src/interfaces/ipc.h new file mode 100644 index 0000000000..e9e6c78053 --- /dev/null +++ b/src/interfaces/ipc.h @@ -0,0 +1,71 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_IPC_H +#define BITCOIN_INTERFACES_IPC_H + +#include <functional> +#include <memory> +#include <typeindex> + +namespace interfaces { +class Init; + +//! Interface providing access to interprocess-communication (IPC) +//! functionality. The IPC implementation is responsible for establishing +//! connections between a controlling process and a process being controlled. +//! When a connection is established, the process being controlled returns an +//! interfaces::Init pointer to the controlling process, which the controlling +//! process can use to get access to other interfaces and functionality. +//! +//! When spawning a new process, the steps are: +//! +//! 1. The controlling process calls interfaces::Ipc::spawnProcess(), which +//! calls ipc::Process::spawn(), which spawns a new process and returns a +//! socketpair file descriptor for communicating with it. +//! interfaces::Ipc::spawnProcess() then calls ipc::Protocol::connect() +//! passing the socketpair descriptor, which returns a local proxy +//! interfaces::Init implementation calling remote interfaces::Init methods. +//! 2. The spawned process calls interfaces::Ipc::startSpawnProcess(), which +//! calls ipc::Process::checkSpawned() to read command line arguments and +//! determine whether it is a spawned process and what socketpair file +//! descriptor it should use. It then calls ipc::Protocol::serve() to handle +//! incoming requests from the socketpair and invoke interfaces::Init +//! interface methods, and exit when the socket is closed. +//! 3. The controlling process calls local proxy interfaces::Init object methods +//! to make other proxy objects calling other remote interfaces. It can also +//! destroy the initial interfaces::Init object to close the connection and +//! shut down the spawned process. +class Ipc +{ +public: + virtual ~Ipc() = default; + + //! Spawn a child process returning pointer to its Init interface. + virtual std::unique_ptr<Init> spawnProcess(const char* exe_name) = 0; + + //! If this is a spawned process, block and handle requests from the parent + //! process by forwarding them to this process's Init interface, then return + //! true. If this is not a spawned child process, return false. + virtual bool startSpawnedProcess(int argc, char* argv[], int& exit_status) = 0; + + //! Add cleanup callback to remote interface that will run when the + //! interface is deleted. + template<typename Interface> + void addCleanup(Interface& iface, std::function<void()> cleanup) + { + addCleanup(typeid(Interface), &iface, std::move(cleanup)); + } + +protected: + //! Internal implementation of public addCleanup method (above) as a + //! type-erased virtual function, since template functions can't be virtual. + virtual void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) = 0; +}; + +//! Return implementation of Ipc interface. +std::unique_ptr<Ipc> MakeIpc(const char* exe_name, const char* process_argv0, Init& init); +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_IPC_H diff --git a/src/ipc/capnp/.gitignore b/src/ipc/capnp/.gitignore new file mode 100644 index 0000000000..036df1430c --- /dev/null +++ b/src/ipc/capnp/.gitignore @@ -0,0 +1,2 @@ +# capnp generated files +*.capnp.* diff --git a/src/ipc/capnp/echo.capnp b/src/ipc/capnp/echo.capnp new file mode 100644 index 0000000000..df36ee0de3 --- /dev/null +++ b/src/ipc/capnp/echo.capnp @@ -0,0 +1,17 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0x888b4f7f51e691f7; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/echo.h"); +$Proxy.include("ipc/capnp/echo.capnp.h"); + +interface Echo $Proxy.wrap("interfaces::Echo") { + destroy @0 (context :Proxy.Context) -> (); + echo @1 (context :Proxy.Context, echo: Text) -> (result :Text); +} diff --git a/src/ipc/capnp/init-types.h b/src/ipc/capnp/init-types.h new file mode 100644 index 0000000000..42031441b5 --- /dev/null +++ b/src/ipc/capnp/init-types.h @@ -0,0 +1,10 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_INIT_TYPES_H +#define BITCOIN_IPC_CAPNP_INIT_TYPES_H + +#include <ipc/capnp/echo.capnp.proxy-types.h> + +#endif // BITCOIN_IPC_CAPNP_INIT_TYPES_H diff --git a/src/ipc/capnp/init.capnp b/src/ipc/capnp/init.capnp new file mode 100644 index 0000000000..e6d358c665 --- /dev/null +++ b/src/ipc/capnp/init.capnp @@ -0,0 +1,20 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xf2c5cfa319406aa6; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/echo.h"); +$Proxy.include("interfaces/init.h"); +$Proxy.includeTypes("ipc/capnp/init-types.h"); + +using Echo = import "echo.capnp"; + +interface Init $Proxy.wrap("interfaces::Init") { + construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); + makeEcho @1 (context :Proxy.Context) -> (result :Echo.Echo); +} diff --git a/src/ipc/capnp/protocol.cpp b/src/ipc/capnp/protocol.cpp new file mode 100644 index 0000000000..74c66c899a --- /dev/null +++ b/src/ipc/capnp/protocol.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/init.h> +#include <ipc/capnp/init.capnp.h> +#include <ipc/capnp/init.capnp.proxy.h> +#include <ipc/capnp/protocol.h> +#include <ipc/exception.h> +#include <ipc/protocol.h> +#include <kj/async.h> +#include <logging.h> +#include <mp/proxy-io.h> +#include <mp/proxy-types.h> +#include <mp/util.h> +#include <util/threadnames.h> + +#include <assert.h> +#include <errno.h> +#include <future> +#include <memory> +#include <mutex> +#include <optional> +#include <string> +#include <thread> + +namespace ipc { +namespace capnp { +namespace { +void IpcLogFn(bool raise, std::string message) +{ + LogPrint(BCLog::IPC, "%s\n", message); + if (raise) throw Exception(message); +} + +class CapnpProtocol : public Protocol +{ +public: + ~CapnpProtocol() noexcept(true) + { + if (m_loop) { + std::unique_lock<std::mutex> lock(m_loop->m_mutex); + m_loop->removeClient(lock); + } + if (m_loop_thread.joinable()) m_loop_thread.join(); + assert(!m_loop); + }; + std::unique_ptr<interfaces::Init> connect(int fd, const char* exe_name) override + { + startLoop(exe_name); + return mp::ConnectStream<messages::Init>(*m_loop, fd); + } + void serve(int fd, const char* exe_name, interfaces::Init& init) override + { + assert(!m_loop); + mp::g_thread_context.thread_name = mp::ThreadName(exe_name); + m_loop.emplace(exe_name, &IpcLogFn, nullptr); + mp::ServeStream<messages::Init>(*m_loop, fd, init); + m_loop->loop(); + m_loop.reset(); + } + void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) override + { + mp::ProxyTypeRegister::types().at(type)(iface).cleanup.emplace_back(std::move(cleanup)); + } + void startLoop(const char* exe_name) + { + if (m_loop) return; + std::promise<void> promise; + m_loop_thread = std::thread([&] { + util::ThreadRename("capnp-loop"); + m_loop.emplace(exe_name, &IpcLogFn, nullptr); + { + std::unique_lock<std::mutex> lock(m_loop->m_mutex); + m_loop->addClient(lock); + } + promise.set_value(); + m_loop->loop(); + m_loop.reset(); + }); + promise.get_future().wait(); + } + std::thread m_loop_thread; + std::optional<mp::EventLoop> m_loop; +}; +} // namespace + +std::unique_ptr<Protocol> MakeCapnpProtocol() { return std::make_unique<CapnpProtocol>(); } +} // namespace capnp +} // namespace ipc diff --git a/src/ipc/capnp/protocol.h b/src/ipc/capnp/protocol.h new file mode 100644 index 0000000000..eb057949d2 --- /dev/null +++ b/src/ipc/capnp/protocol.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_PROTOCOL_H +#define BITCOIN_IPC_CAPNP_PROTOCOL_H + +#include <memory> + +namespace ipc { +class Protocol; +namespace capnp { +std::unique_ptr<Protocol> MakeCapnpProtocol(); +} // namespace capnp +} // namespace ipc + +#endif // BITCOIN_IPC_CAPNP_PROTOCOL_H diff --git a/src/ipc/exception.h b/src/ipc/exception.h new file mode 100644 index 0000000000..53dee8124a --- /dev/null +++ b/src/ipc/exception.h @@ -0,0 +1,20 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_EXCEPTION_H +#define BITCOIN_IPC_EXCEPTION_H + +#include <stdexcept> + +namespace ipc { +//! Exception class thrown when a call to remote method fails due to an IPC +//! error, like a socket getting disconnected. +class Exception : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; +} // namespace ipc + +#endif // BITCOIN_IPC_EXCEPTION_H diff --git a/src/ipc/interfaces.cpp b/src/ipc/interfaces.cpp new file mode 100644 index 0000000000..ad4b78ed81 --- /dev/null +++ b/src/ipc/interfaces.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> +#include <ipc/capnp/protocol.h> +#include <ipc/process.h> +#include <ipc/protocol.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/system.h> + +#include <functional> +#include <memory> +#include <stdexcept> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <string> +#include <unistd.h> +#include <utility> +#include <vector> + +namespace ipc { +namespace { +class IpcImpl : public interfaces::Ipc +{ +public: + IpcImpl(const char* exe_name, const char* process_argv0, interfaces::Init& init) + : m_exe_name(exe_name), m_process_argv0(process_argv0), m_init(init), + m_protocol(ipc::capnp::MakeCapnpProtocol()), m_process(ipc::MakeProcess()) + { + } + std::unique_ptr<interfaces::Init> spawnProcess(const char* new_exe_name) override + { + int pid; + int fd = m_process->spawn(new_exe_name, m_process_argv0, pid); + LogPrint(::BCLog::IPC, "Process %s pid %i launched\n", new_exe_name, pid); + auto init = m_protocol->connect(fd, m_exe_name); + Ipc::addCleanup(*init, [this, new_exe_name, pid] { + int status = m_process->waitSpawned(pid); + LogPrint(::BCLog::IPC, "Process %s pid %i exited with status %i\n", new_exe_name, pid, status); + }); + return init; + } + bool startSpawnedProcess(int argc, char* argv[], int& exit_status) override + { + exit_status = EXIT_FAILURE; + int32_t fd = -1; + if (!m_process->checkSpawned(argc, argv, fd)) { + return false; + } + m_protocol->serve(fd, m_exe_name, m_init); + exit_status = EXIT_SUCCESS; + return true; + } + void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) override + { + m_protocol->addCleanup(type, iface, std::move(cleanup)); + } + const char* m_exe_name; + const char* m_process_argv0; + interfaces::Init& m_init; + std::unique_ptr<Protocol> m_protocol; + std::unique_ptr<Process> m_process; +}; +} // namespace +} // namespace ipc + +namespace interfaces { +std::unique_ptr<Ipc> MakeIpc(const char* exe_name, const char* process_argv0, Init& init) +{ + return std::make_unique<ipc::IpcImpl>(exe_name, process_argv0, init); +} +} // namespace interfaces diff --git a/src/ipc/process.cpp b/src/ipc/process.cpp new file mode 100644 index 0000000000..43ed1f1bae --- /dev/null +++ b/src/ipc/process.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> +#include <ipc/process.h> +#include <ipc/protocol.h> +#include <mp/util.h> +#include <tinyformat.h> +#include <util/strencodings.h> + +#include <cstdint> +#include <exception> +#include <iostream> +#include <stdexcept> +#include <stdlib.h> +#include <string.h> +#include <system_error> +#include <unistd.h> +#include <utility> +#include <vector> + +namespace ipc { +namespace { +class ProcessImpl : public Process +{ +public: + int spawn(const std::string& new_exe_name, const fs::path& argv0_path, int& pid) override + { + return mp::SpawnProcess(pid, [&](int fd) { + fs::path path = argv0_path; + path.remove_filename(); + path.append(new_exe_name); + return std::vector<std::string>{path.string(), "-ipcfd", strprintf("%i", fd)}; + }); + } + int waitSpawned(int pid) override { return mp::WaitProcess(pid); } + bool checkSpawned(int argc, char* argv[], int& fd) override + { + // If this process was not started with a single -ipcfd argument, it is + // not a process spawned by the spawn() call above, so return false and + // do not try to serve requests. + if (argc != 3 || strcmp(argv[1], "-ipcfd") != 0) { + return false; + } + // If a single -ipcfd argument was provided, return true and get the + // file descriptor so Protocol::serve() can be called to handle + // requests from the parent process. The -ipcfd argument is not valid + // in combination with other arguments because the parent process + // should be able to control the child process through the IPC protocol + // without passing information out of band. + if (!ParseInt32(argv[2], &fd)) { + throw std::runtime_error(strprintf("Invalid -ipcfd number '%s'", argv[2])); + } + return true; + } +}; +} // namespace + +std::unique_ptr<Process> MakeProcess() { return std::make_unique<ProcessImpl>(); } +} // namespace ipc diff --git a/src/ipc/process.h b/src/ipc/process.h new file mode 100644 index 0000000000..4bb2930d9c --- /dev/null +++ b/src/ipc/process.h @@ -0,0 +1,42 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_PROCESS_H +#define BITCOIN_IPC_PROCESS_H + +#include <memory> +#include <string> + +namespace ipc { +class Protocol; + +//! IPC process interface for spawning bitcoin processes and serving requests +//! in processes that have been spawned. +//! +//! There will be different implementations of this interface depending on the +//! platform (e.g. unix, windows). +class Process +{ +public: + virtual ~Process() = default; + + //! Spawn process and return socket file descriptor for communicating with + //! it. + virtual int spawn(const std::string& new_exe_name, const fs::path& argv0_path, int& pid) = 0; + + //! Wait for spawned process to exit and return its exit code. + virtual int waitSpawned(int pid) = 0; + + //! Parse command line and determine if current process is a spawned child + //! process. If so, return true and a file descriptor for communicating + //! with the parent process. + virtual bool checkSpawned(int argc, char* argv[], int& fd) = 0; +}; + +//! Constructor for Process interface. Implementation will vary depending on +//! the platform (unix or windows). +std::unique_ptr<Process> MakeProcess(); +} // namespace ipc + +#endif // BITCOIN_IPC_PROCESS_H diff --git a/src/ipc/protocol.h b/src/ipc/protocol.h new file mode 100644 index 0000000000..af955b0007 --- /dev/null +++ b/src/ipc/protocol.h @@ -0,0 +1,39 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_PROTOCOL_H +#define BITCOIN_IPC_PROTOCOL_H + +#include <interfaces/init.h> + +#include <functional> +#include <memory> +#include <typeindex> + +namespace ipc { +//! IPC protocol interface for calling IPC methods over sockets. +//! +//! There may be different implementations of this interface for different IPC +//! protocols (e.g. Cap'n Proto, gRPC, JSON-RPC, or custom protocols). +class Protocol +{ +public: + virtual ~Protocol() = default; + + //! Return Init interface that forwards requests over given socket descriptor. + //! Socket communication is handled on a background thread. + virtual std::unique_ptr<interfaces::Init> connect(int fd, const char* exe_name) = 0; + + //! Handle requests on provided socket descriptor, forwarding them to the + //! provided Init interface. Socket communication is handled on the + //! current thread, and this call blocks until the socket is closed. + virtual void serve(int fd, const char* exe_name, interfaces::Init& init) = 0; + + //! Add cleanup callback to interface that will run when the interface is + //! deleted. + virtual void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) = 0; +}; +} // namespace ipc + +#endif // BITCOIN_IPC_PROTOCOL_H diff --git a/src/logging.cpp b/src/logging.cpp index 866213786e..e5187fd596 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -157,6 +157,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::LEVELDB, "leveldb"}, {BCLog::VALIDATION, "validation"}, {BCLog::I2P, "i2p"}, + {BCLog::IPC, "ipc"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index 436f0cd12e..d04bc99268 100644 --- a/src/logging.h +++ b/src/logging.h @@ -58,6 +58,7 @@ namespace BCLog { LEVELDB = (1 << 20), VALIDATION = (1 << 21), I2P = (1 << 22), + IPC = (1 << 23), ALL = ~(uint32_t)0, }; diff --git a/src/net.cpp b/src/net.cpp index 5aa267f0d7..fe1a0dfded 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2635,23 +2635,26 @@ void CConnman::StopNodes() } } - // Close sockets - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) + // Delete peer connections. + std::vector<CNode*> nodes; + WITH_LOCK(cs_vNodes, nodes.swap(vNodes)); + for (CNode* pnode : nodes) { pnode->CloseSocketDisconnect(); - for (ListenSocket& hListenSocket : vhListenSocket) - if (hListenSocket.socket != INVALID_SOCKET) - if (!CloseSocket(hListenSocket.socket)) - LogPrintf("CloseSocket(hListenSocket) failed with error %s\n", NetworkErrorString(WSAGetLastError())); - - // clean up some globals (to help leak detection) - for (CNode* pnode : vNodes) { DeleteNode(pnode); } + + // Close listening sockets. + for (ListenSocket& hListenSocket : vhListenSocket) { + if (hListenSocket.socket != INVALID_SOCKET) { + if (!CloseSocket(hListenSocket.socket)) { + LogPrintf("CloseSocket(hListenSocket) failed with error %s\n", NetworkErrorString(WSAGetLastError())); + } + } + } + for (CNode* pnode : vNodesDisconnected) { DeleteNode(pnode); } - vNodes.clear(); vNodesDisconnected.clear(); vhListenSocket.clear(); semOutbound.reset(); diff --git a/src/node/context.h b/src/node/context.h index 2be9a584e6..06adb33a80 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -22,6 +22,7 @@ class PeerManager; namespace interfaces { class Chain; class ChainClient; +class Init; class WalletClient; } // namespace interfaces @@ -36,6 +37,8 @@ class WalletClient; //! any member functions. It should just be a collection of references that can //! be used without pulling in unwanted dependencies or functionality. struct NodeContext { + //! Init interface for initializing current process and connecting to other processes. + interfaces::Init* init{nullptr}; std::unique_ptr<CAddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index f2c555de52..04161020b2 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -8,6 +8,7 @@ #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/peertablemodel.h> +#include <qt/peertablesortproxy.h> #include <clientversion.h> #include <interfaces/handler.h> @@ -38,7 +39,11 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO { cachedBestHeaderHeight = -1; cachedBestHeaderTime = -1; + peerTableModel = new PeerTableModel(m_node, this); + m_peer_table_sort_proxy = new PeerTableSortProxy(this); + m_peer_table_sort_proxy->setSourceModel(peerTableModel); + banTableModel = new BanTableModel(m_node, this); QTimer* timer = new QTimer; @@ -184,6 +189,11 @@ PeerTableModel *ClientModel::getPeerTableModel() return peerTableModel; } +PeerTableSortProxy* ClientModel::peerTableSortProxy() +{ + return m_peer_table_sort_proxy; +} + BanTableModel *ClientModel::getBanTableModel() { return banTableModel; diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 7ac4cc040b..7a199ef19c 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -17,6 +17,7 @@ class BanTableModel; class CBlockIndex; class OptionsModel; class PeerTableModel; +class PeerTableSortProxy; enum class SynchronizationState; namespace interfaces { @@ -54,6 +55,7 @@ public: interfaces::Node& node() const { return m_node; } OptionsModel *getOptionsModel(); PeerTableModel *getPeerTableModel(); + PeerTableSortProxy* peerTableSortProxy(); BanTableModel *getBanTableModel(); //! Return number of connections, default is in- and outbound (total) @@ -96,6 +98,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_notify_header_tip; OptionsModel *optionsModel; PeerTableModel *peerTableModel; + PeerTableSortProxy* m_peer_table_sort_proxy{nullptr}; BanTableModel *banTableModel; //! A thread to interact with m_node asynchronously diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 3459bf4cf8..6c4e326011 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -11,56 +11,19 @@ #include <utility> -#include <QDebug> #include <QList> #include <QTimer> -bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombinedStats &right) const -{ - const CNodeStats *pLeft = &(left.nodeStats); - const CNodeStats *pRight = &(right.nodeStats); - - if (order == Qt::DescendingOrder) - std::swap(pLeft, pRight); - - switch (static_cast<PeerTableModel::ColumnIndex>(column)) { - case PeerTableModel::NetNodeId: - return pLeft->nodeid < pRight->nodeid; - case PeerTableModel::Address: - return pLeft->addrName.compare(pRight->addrName) < 0; - case PeerTableModel::ConnectionType: - return pLeft->m_conn_type < pRight->m_conn_type; - case PeerTableModel::Network: - return pLeft->m_network < pRight->m_network; - case PeerTableModel::Ping: - return pLeft->m_min_ping_time < pRight->m_min_ping_time; - case PeerTableModel::Sent: - return pLeft->nSendBytes < pRight->nSendBytes; - case PeerTableModel::Received: - return pLeft->nRecvBytes < pRight->nRecvBytes; - case PeerTableModel::Subversion: - return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; - } // no default case, so the compiler can warn about missing cases - assert(false); -} - // private implementation class PeerTablePriv { public: /** Local cache of peer information */ QList<CNodeCombinedStats> cachedNodeStats; - /** Column to sort nodes by (default to unsorted) */ - int sortColumn{-1}; - /** Order (ascending or descending) to sort nodes by */ - Qt::SortOrder sortOrder; - /** Index of rows by node ID */ - std::map<NodeId, int> mapNodeRows; /** Pull a full list of peers from vNodes into our cache */ void refreshPeers(interfaces::Node& node) { - { cachedNodeStats.clear(); interfaces::Node::NodesStats nodes_stats; @@ -74,17 +37,6 @@ public: stats.nodeStateStats = std::get<2>(node_stats); cachedNodeStats.append(stats); } - } - - if (sortColumn >= 0) - // sort cacheNodeStats (use stable sort to prevent rows jumping around unnecessarily) - std::stable_sort(cachedNodeStats.begin(), cachedNodeStats.end(), NodeLessThan(sortColumn, sortOrder)); - - // build index map - mapNodeRows.clear(); - int row = 0; - for (const CNodeCombinedStats& stats : cachedNodeStats) - mapNodeRows.insert(std::pair<NodeId, int>(stats.nodeStats.nodeid, row++)); } int size() const @@ -194,10 +146,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const } // no default case, so the compiler can warn about missing cases assert(false); } else if (role == StatsRole) { - switch (index.column()) { - case NetNodeId: return QVariant::fromValue(rec); - default: return QVariant(); - } + return QVariant::fromValue(rec); } return QVariant(); @@ -239,19 +188,3 @@ void PeerTableModel::refresh() priv->refreshPeers(m_node); Q_EMIT layoutChanged(); } - -int PeerTableModel::getRowByNodeId(NodeId nodeid) -{ - std::map<NodeId, int>::iterator it = priv->mapNodeRows.find(nodeid); - if (it == priv->mapNodeRows.end()) - return -1; - - return it->second; -} - -void PeerTableModel::sort(int column, Qt::SortOrder order) -{ - priv->sortColumn = column; - priv->sortOrder = order; - refresh(); -} diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0823235ec0..9c7bc25da2 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -30,18 +30,6 @@ struct CNodeCombinedStats { }; Q_DECLARE_METATYPE(CNodeCombinedStats*) -class NodeLessThan -{ -public: - NodeLessThan(int nColumn, Qt::SortOrder fOrder) : - column(nColumn), order(fOrder) {} - bool operator()(const CNodeCombinedStats &left, const CNodeCombinedStats &right) const; - -private: - int column; - Qt::SortOrder order; -}; - /** Qt model providing information about connected peers, similar to the "getpeerinfo" RPC call. Used by the rpc console UI. @@ -53,7 +41,6 @@ class PeerTableModel : public QAbstractTableModel public: explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); - int getRowByNodeId(NodeId nodeid); void startAutoRefresh(); void stopAutoRefresh(); @@ -80,7 +67,6 @@ public: QVariant headerData(int section, Qt::Orientation orientation, int role) const override; QModelIndex index(int row, int column, const QModelIndex &parent) const override; Qt::ItemFlags flags(const QModelIndex &index) const override; - void sort(int column, Qt::SortOrder order) override; /*@}*/ public Q_SLOTS: diff --git a/src/qt/peertablesortproxy.cpp b/src/qt/peertablesortproxy.cpp new file mode 100644 index 0000000000..78932da8d4 --- /dev/null +++ b/src/qt/peertablesortproxy.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <qt/peertablesortproxy.h> + +#include <qt/peertablemodel.h> +#include <util/check.h> + +#include <QModelIndex> +#include <QString> +#include <QVariant> + +PeerTableSortProxy::PeerTableSortProxy(QObject* parent) + : QSortFilterProxyModel(parent) +{ +} + +bool PeerTableSortProxy::lessThan(const QModelIndex& left_index, const QModelIndex& right_index) const +{ + const CNodeStats left_stats = Assert(sourceModel()->data(left_index, PeerTableModel::StatsRole).value<CNodeCombinedStats*>())->nodeStats; + const CNodeStats right_stats = Assert(sourceModel()->data(right_index, PeerTableModel::StatsRole).value<CNodeCombinedStats*>())->nodeStats; + + switch (static_cast<PeerTableModel::ColumnIndex>(left_index.column())) { + case PeerTableModel::NetNodeId: + return left_stats.nodeid < right_stats.nodeid; + case PeerTableModel::Address: + return left_stats.addrName.compare(right_stats.addrName) < 0; + case PeerTableModel::ConnectionType: + return left_stats.m_conn_type < right_stats.m_conn_type; + case PeerTableModel::Network: + return left_stats.m_network < right_stats.m_network; + case PeerTableModel::Ping: + return left_stats.m_min_ping_time < right_stats.m_min_ping_time; + case PeerTableModel::Sent: + return left_stats.nSendBytes < right_stats.nSendBytes; + case PeerTableModel::Received: + return left_stats.nRecvBytes < right_stats.nRecvBytes; + case PeerTableModel::Subversion: + return left_stats.cleanSubVer.compare(right_stats.cleanSubVer) < 0; + } // no default case, so the compiler can warn about missing cases + assert(false); +} diff --git a/src/qt/peertablesortproxy.h b/src/qt/peertablesortproxy.h new file mode 100644 index 0000000000..1879f6b400 --- /dev/null +++ b/src/qt/peertablesortproxy.h @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_PEERTABLESORTPROXY_H +#define BITCOIN_QT_PEERTABLESORTPROXY_H + +#include <QSortFilterProxyModel> + +QT_BEGIN_NAMESPACE +class QModelIndex; +QT_END_NAMESPACE + +class PeerTableSortProxy : public QSortFilterProxyModel +{ + Q_OBJECT + +public: + explicit PeerTableSortProxy(QObject* parent = nullptr); + +protected: + bool lessThan(const QModelIndex& left_index, const QModelIndex& right_index) const override; +}; + +#endif // BITCOIN_QT_PEERTABLESORTPROXY_H diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 85412ca75b..006f60e7a1 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -9,13 +9,14 @@ #include <qt/rpcconsole.h> #include <qt/forms/ui_debugwindow.h> +#include <chainparams.h> +#include <interfaces/node.h> +#include <netbase.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> +#include <qt/peertablesortproxy.h> #include <qt/platformstyle.h> #include <qt/walletmodel.h> -#include <chainparams.h> -#include <interfaces/node.h> -#include <netbase.h> #include <rpc/client.h> #include <rpc/server.h> #include <util/strencodings.h> @@ -606,7 +607,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ connect(model, &ClientModel::mempoolSizeChanged, this, &RPCConsole::setMempoolSize); // set up peer table - ui->peerWidget->setModel(model->getPeerTableModel()); + ui->peerWidget->setModel(model->peerTableSortProxy()); ui->peerWidget->verticalHeader()->hide(); ui->peerWidget->setSelectionBehavior(QAbstractItemView::SelectRows); ui->peerWidget->setSelectionMode(QAbstractItemView::ExtendedSelection); @@ -627,10 +628,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // peer table signal handling - update peer details when selecting new node connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::updateDetailWidget); - // peer table signal handling - update peer details when new nodes are added to the model - connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::peerLayoutChanged); - // peer table signal handling - cache selected node ids - connect(model->getPeerTableModel(), &PeerTableModel::layoutAboutToBeChanged, this, &RPCConsole::peerLayoutAboutToChange); + connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::updateDetailWidget); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); @@ -1014,67 +1012,6 @@ void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } -void RPCConsole::peerLayoutAboutToChange() -{ - cachedNodeids.clear(); - for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { - const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); - cachedNodeids.append(stats->nodeStats.nodeid); - } -} - -void RPCConsole::peerLayoutChanged() -{ - if (!clientModel || !clientModel->getPeerTableModel()) - return; - - bool fUnselect = false; - bool fReselect = false; - - if (cachedNodeids.empty()) // no node selected yet - return; - - // find the currently selected row - int selectedRow = -1; - QModelIndexList selectedModelIndex = ui->peerWidget->selectionModel()->selectedIndexes(); - if (!selectedModelIndex.isEmpty()) { - selectedRow = selectedModelIndex.first().row(); - } - - // check if our detail node has a row in the table (it may not necessarily - // be at selectedRow since its position can change after a layout change) - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.first()); - - if (detailNodeRow < 0) - { - // detail node disappeared from table (node disconnected) - fUnselect = true; - } - else - { - if (detailNodeRow != selectedRow) - { - // detail node moved position - fUnselect = true; - fReselect = true; - } - } - - if (fUnselect && selectedRow >= 0) { - clearSelectedNode(); - } - - if (fReselect) - { - for(int i = 0; i < cachedNodeids.size(); i++) - { - ui->peerWidget->selectRow(clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.at(i))); - } - } - - updateDetailWidget(); -} - void RPCConsole::updateDetailWidget() { const QList<QModelIndex> selected_peers = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index b9806e40c9..5182d60a0d 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -118,10 +118,6 @@ public Q_SLOTS: void browseHistory(int offset); /** Scroll console view to end */ void scrollToEnd(); - /** Handle selection caching before update */ - void peerLayoutAboutToChange(); - /** Handle updated peer information */ - void peerLayoutChanged(); /** Disconnect a selected node on the Peers tab */ void disconnectSelectedNode(); /** Ban a selected node on the Peers tab */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index f0e720617c..e1a4faa266 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -271,7 +271,7 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa m_current_transaction = std::make_unique<WalletModelTransaction>(recipients); WalletModel::SendCoinsReturn prepareStatus; - updateCoinControlState(*m_coin_control); + updateCoinControlState(); prepareStatus = model->prepareTransaction(*m_current_transaction, *m_coin_control); @@ -740,19 +740,19 @@ void SendCoinsDialog::updateFeeMinimizedLabel() } } -void SendCoinsDialog::updateCoinControlState(CCoinControl& ctrl) +void SendCoinsDialog::updateCoinControlState() { if (ui->radioCustomFee->isChecked()) { - ctrl.m_feerate = CFeeRate(ui->customFee->value()); + m_coin_control->m_feerate = CFeeRate(ui->customFee->value()); } else { - ctrl.m_feerate.reset(); + m_coin_control->m_feerate.reset(); } // Avoid using global defaults when sending money from the GUI // Either custom fee will be used or if not selected, the confirmation target from dropdown box - ctrl.m_confirm_target = getConfTargetForIndex(ui->confTargetSelector->currentIndex()); - ctrl.m_signal_bip125_rbf = ui->optInRBF->isChecked(); + m_coin_control->m_confirm_target = getConfTargetForIndex(ui->confTargetSelector->currentIndex()); + m_coin_control->m_signal_bip125_rbf = ui->optInRBF->isChecked(); // Include watch-only for wallets without private key - ctrl.fAllowWatchOnly = model->wallet().privateKeysDisabled(); + m_coin_control->fAllowWatchOnly = model->wallet().privateKeysDisabled(); } void SendCoinsDialog::updateNumberOfBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers, SynchronizationState sync_state) { @@ -765,7 +765,7 @@ void SendCoinsDialog::updateSmartFeeLabel() { if(!model || !model->getOptionsModel()) return; - updateCoinControlState(*m_coin_control); + updateCoinControlState(); m_coin_control->m_feerate.reset(); // Explicitly use only fee estimation rate for smart fee labels int returned_target; FeeReason reason; @@ -839,8 +839,9 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) { ui->frameCoinControl->setVisible(checked); - if (!checked && model) // coin control features disabled - m_coin_control->SetNull(); + if (!checked && model) { // coin control features disabled + m_coin_control = std::make_unique<CCoinControl>(); + } coinControlUpdateLabels(); } @@ -928,7 +929,7 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - updateCoinControlState(*m_coin_control); + updateCoinControlState(); // set pay amounts CoinControlDialog::payAmounts.clear(); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 3e276201ba..33736f8095 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -76,8 +76,7 @@ private: // Format confirmation message bool PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text); void updateFeeMinimizedLabel(); - // Update the passed in CCoinControl with state from the GUI - void updateCoinControlState(CCoinControl& ctrl); + void updateCoinControlState(); private Q_SLOTS: void sendButtonClicked(bool checked); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e7fd97ee1f..d36814716b 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1239,7 +1239,8 @@ static RPCHelpMan verifychain() LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); - return CVerifyDB().VerifyDB(Params(), active_chainstate, &active_chainstate.CoinsTip(), check_level, check_depth); + return CVerifyDB().VerifyDB( + active_chainstate, Params(), active_chainstate.CoinsTip(), check_level, check_depth); }, }; } diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 00a06260ea..09b32345a2 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -7,6 +7,9 @@ #include <index/blockfilterindex.h> #include <index/txindex.h> #include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> #include <key_io.h> #include <node/context.h> #include <outputtype.h> @@ -644,6 +647,43 @@ static RPCHelpMan echo(const std::string& name) static RPCHelpMan echo() { return echo("echo"); } static RPCHelpMan echojson() { return echo("echojson"); } +static RPCHelpMan echoipc() +{ + return RPCHelpMan{ + "echoipc", + "\nEcho back the input argument, passing it through a spawned process in a multiprocess build.\n" + "This command is for testing.\n", + {{"arg", RPCArg::Type::STR, RPCArg::Optional::NO, "The string to echo",}}, + RPCResult{RPCResult::Type::STR, "echo", "The echoed string."}, + RPCExamples{HelpExampleCli("echo", "\"Hello world\"") + + HelpExampleRpc("echo", "\"Hello world\"")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + std::unique_ptr<interfaces::Echo> echo; + if (interfaces::Ipc* ipc = Assert(EnsureAnyNodeContext(request.context).init)->ipc()) { + // Spawn a new bitcoin-node process and call makeEcho to get a + // client pointer to a interfaces::Echo instance running in + // that process. This is just for testing. A slightly more + // realistic test spawning a different executable instead of + // the same executable would add a new bitcoin-echo executable, + // and spawn bitcoin-echo below instead of bitcoin-node. But + // using bitcoin-node avoids the need to build and install a + // new executable just for this one test. + auto init = ipc->spawnProcess("bitcoin-node"); + echo = init->makeEcho(); + ipc->addCleanup(*echo, [init = init.release()] { delete init; }); + } else { + // IPC support is not available because this is a bitcoind + // process not a bitcoind-node process, so just create a local + // interfaces::Echo object and return it so the `echoipc` RPC + // method will work, and the python test calling `echoipc` + // can expect the same result. + echo = interfaces::MakeEcho(); + } + return echo->echo(request.params[0].get_str()); + }, + }; +} + static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_name) { UniValue ret_summary(UniValue::VOBJ); @@ -719,6 +759,7 @@ static const CRPCCommand commands[] = { "hidden", &mockscheduler, }, { "hidden", &echo, }, { "hidden", &echojson, }, + { "hidden", &echoipc, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 4ddbc8338e..7b661a0d1d 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK_EQUAL(script.size(), 25U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); @@ -89,7 +89,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) script << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK_EQUAL(script.size(), 23U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); @@ -107,7 +107,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) BOOST_CHECK_EQUAL(script.size(), 35U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); @@ -124,7 +124,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) BOOST_CHECK_EQUAL(script.size(), 67U); // 1 char code + 65 char pubkey + OP_CHECKSIG - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 96e1cfa08f..7b99193ad0 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -100,7 +100,6 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE g_setup->m_node.peerman->SendMessages(&p2p_node); } SyncWithValidationInterfaceQueue(); - LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 203c0ef8e1..11b236c9bd 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -80,6 +80,5 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) } } SyncWithValidationInterfaceQueue(); - LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp new file mode 100644 index 0000000000..dae6f6b6a7 --- /dev/null +++ b/src/test/fuzz/rpc.cpp @@ -0,0 +1,378 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <base58.h> +#include <chainparamsbase.h> +#include <core_io.h> +#include <interfaces/chain.h> +#include <key.h> +#include <key_io.h> +#include <node/context.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <psbt.h> +#include <rpc/blockchain.h> +#include <rpc/client.h> +#include <rpc/request.h> +#include <rpc/server.h> +#include <rpc/util.h> +#include <span.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <univalue.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/time.h> + +#include <cstdint> +#include <iostream> +#include <memory> +#include <optional> +#include <stdexcept> +#include <string> +#include <vector> + +namespace { +struct RPCFuzzTestingSetup : public TestingSetup { + RPCFuzzTestingSetup(const std::string& chain_name, const std::vector<const char*>& extra_args) : TestingSetup{chain_name, extra_args} + { + } + + UniValue CallRPC(const std::string& rpc_method, const std::vector<std::string>& arguments) + { + JSONRPCRequest request; + request.context = &m_node; + request.strMethod = rpc_method; + request.params = RPCConvertValues(rpc_method, arguments); + return tableRPC.execute(request); + } + + std::vector<std::string> GetRPCCommands() const + { + return tableRPC.listCommands(); + } +}; + +RPCFuzzTestingSetup* rpc_testing_setup = nullptr; +std::string g_limit_to_rpc_command; + +// RPC commands which are not appropriate for fuzzing: such as RPC commands +// reading or writing to a filename passed as an RPC parameter, RPC commands +// resulting in network activity, etc. +const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ + "addconnection", // avoid DNS lookups + "addnode", // avoid DNS lookups + "addpeeraddress", // avoid DNS lookups + "analyzepsbt", // avoid signed integer overflow in CFeeRate::GetFee(unsigned long) (https://github.com/bitcoin/bitcoin/issues/20607) + "dumptxoutset", // avoid writing to disk +#ifdef ENABLE_WALLET + "dumpwallet", // avoid writing to disk +#endif + "echoipc", // avoid assertion failure (Assertion `"EnsureAnyNodeContext(request.context).init" && check' failed.) + "generatetoaddress", // avoid timeout + "gettxoutproof", // avoid slow execution +#ifdef ENABLE_WALLET + "importwallet", // avoid reading from disk + "loadwallet", // avoid reading from disk +#endif + "mockscheduler", // avoid assertion failure (Assertion `delta_seconds.count() > 0 && delta_seconds < std::chrono::hours{1}' failed.) + "prioritisetransaction", // avoid signed integer overflow in CTxMemPool::PrioritiseTransaction(uint256 const&, long const&) (https://github.com/bitcoin/bitcoin/issues/20626) + "setban", // avoid DNS lookups + "stop", // avoid shutdown state +}; + +// RPC commands which are safe for fuzzing. +const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ + "clearbanned", + "combinepsbt", + "combinerawtransaction", + "converttopsbt", + "createmultisig", + "createpsbt", + "createrawtransaction", + "decodepsbt", + "decoderawtransaction", + "decodescript", + "deriveaddresses", + "disconnectnode", + "echo", + "echojson", + "estimaterawfee", + "estimatesmartfee", + "finalizepsbt", + "generate", + "generateblock", + "generatetodescriptor", + "getaddednodeinfo", + "getbestblockhash", + "getblock", + "getblockchaininfo", + "getblockcount", + "getblockfilter", + "getblockhash", + "getblockheader", + "getblockstats", + "getblocktemplate", + "getchaintips", + "getchaintxstats", + "getconnectioncount", + "getdescriptorinfo", + "getdifficulty", + "getindexinfo", + "getmemoryinfo", + "getmempoolancestors", + "getmempooldescendants", + "getmempoolentry", + "getmempoolinfo", + "getmininginfo", + "getnettotals", + "getnetworkhashps", + "getnetworkinfo", + "getnodeaddresses", + "getpeerinfo", + "getrawmempool", + "getrawtransaction", + "getrpcinfo", + "gettxout", + "gettxoutsetinfo", + "help", + "invalidateblock", + "joinpsbts", + "listbanned", + "logging", + "ping", + "preciousblock", + "pruneblockchain", + "reconsiderblock", + "savemempool", + "scantxoutset", + "sendrawtransaction", + "setmocktime", + "setnetworkactive", + "signmessagewithprivkey", + "signrawtransactionwithkey", + "submitblock", + "submitheader", + "syncwithvalidationinterfacequeue", + "testmempoolaccept", + "uptime", + "utxoupdatepsbt", + "validateaddress", + "verifychain", + "verifymessage", + "verifytxoutproof", + "waitforblock", + "waitforblockheight", + "waitfornewblock", +}; + +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + const size_t max_string_length = 4096; + std::string r; + CallOneOf( + fuzzed_data_provider, + [&] { + // string argument + r = fuzzed_data_provider.ConsumeRandomLengthString(max_string_length); + }, + [&] { + // base64 argument + r = EncodeBase64(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // hex argument + r = HexStr(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // bool argument + r = fuzzed_data_provider.ConsumeBool() ? "true" : "false"; + }, + [&] { + // range argument + r = "[" + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "," + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "]"; + }, + [&] { + // integral argument (int64_t) + r = ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()); + }, + [&] { + // integral argument (uint64_t) + r = ToString(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + // floating point argument + r = strprintf("%f", fuzzed_data_provider.ConsumeFloatingPoint<double>()); + }, + [&] { + // tx destination argument + r = EncodeDestination(ConsumeTxDestination(fuzzed_data_provider)); + }, + [&] { + // uint160 argument + r = ConsumeUInt160(fuzzed_data_provider).ToString(); + }, + [&] { + // uint256 argument + r = ConsumeUInt256(fuzzed_data_provider).ToString(); + }, + [&] { + // base32 argument + r = EncodeBase32(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // base58 argument + r = EncodeBase58(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length))); + }, + [&] { + // base58 argument with checksum + r = EncodeBase58Check(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length))); + }, + [&] { + // hex encoded block + std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + if (!opt_block) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_block; + r = HexStr(data_stream); + }, + [&] { + // hex encoded block header + std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + if (!opt_block_header) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_block_header; + r = HexStr(data_stream); + }, + [&] { + // hex encoded tx + std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!opt_tx) { + return; + } + CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; + data_stream << *opt_tx; + r = HexStr(data_stream); + }, + [&] { + // base64 encoded psbt + std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); + if (!opt_psbt) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_psbt; + r = EncodeBase64({data_stream.begin(), data_stream.end()}); + }, + [&] { + // base58 encoded key + const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32); + CKey key; + key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + if (!key.IsValid()) { + return; + } + r = EncodeSecret(key); + }, + [&] { + // hex encoded pubkey + const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32); + CKey key; + key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + if (!key.IsValid()) { + return; + } + r = HexStr(key.GetPubKey()); + }); + return r; +} + +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + std::vector<std::string> scalar_arguments; + while (fuzzed_data_provider.ConsumeBool()) { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + } + return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; +} + +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); +} + +RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() +{ + static const auto setup = MakeNoLogFileContext<RPCFuzzTestingSetup>(); + SetRPCWarmupFinished(); + return setup.get(); +} +}; // namespace + +void initialize_rpc() +{ + rpc_testing_setup = InitializeRPCFuzzTestingSetup(); + const std::vector<std::string> supported_rpc_commands = rpc_testing_setup->GetRPCCommands(); + for (const std::string& rpc_command : supported_rpc_commands) { + const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end(); + const bool not_safe_for_fuzzing = std::find(RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(); + if (!(safe_for_fuzzing || not_safe_for_fuzzing)) { + std::cerr << "Error: RPC command \"" << rpc_command << "\" not found in RPC_COMMANDS_SAFE_FOR_FUZZING or RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + if (safe_for_fuzzing && not_safe_for_fuzzing) { + std::cerr << "Error: RPC command \"" << rpc_command << "\" found in *both* RPC_COMMANDS_SAFE_FOR_FUZZING and RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + for (const std::string& rpc_command : RPC_COMMANDS_SAFE_FOR_FUZZING) { + const bool supported_rpc_command = std::find(supported_rpc_commands.begin(), supported_rpc_commands.end(), rpc_command) != supported_rpc_commands.end(); + if (!supported_rpc_command) { + std::cerr << "Error: Unknown RPC command \"" << rpc_command << "\" found in RPC_COMMANDS_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + for (const std::string& rpc_command : RPC_COMMANDS_NOT_SAFE_FOR_FUZZING) { + const bool supported_rpc_command = std::find(supported_rpc_commands.begin(), supported_rpc_commands.end(), rpc_command) != supported_rpc_commands.end(); + if (!supported_rpc_command) { + std::cerr << "Error: Unknown RPC command \"" << rpc_command << "\" found in RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + const char* limit_to_rpc_command_env = std::getenv("LIMIT_TO_RPC_COMMAND"); + if (limit_to_rpc_command_env != nullptr) { + g_limit_to_rpc_command = std::string{limit_to_rpc_command_env}; + } +} + +FUZZ_TARGET_INIT(rpc, initialize_rpc) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + SetMockTime(ConsumeTime(fuzzed_data_provider)); + const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); + if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { + return; + } + const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end(); + if (!safe_for_fuzzing) { + return; + } + std::vector<std::string> arguments; + while (fuzzed_data_provider.ConsumeBool()) { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + } + try { + rpc_testing_setup->CallRPC(rpc_command, arguments); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } +} diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index e87ae5b04b..b87bcf2ef5 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -43,7 +43,7 @@ FUZZ_TARGET_INIT(script, initialize_script) if (!script_opt) return; const CScript script{*script_opt}; - std::vector<unsigned char> compressed; + CompressedScript compressed; if (CompressScript(script, compressed)) { const unsigned int size = compressed[0]; compressed.erase(compressed.begin()); @@ -55,22 +55,45 @@ FUZZ_TARGET_INIT(script, initialize_script) } CTxDestination address; - (void)ExtractDestination(script, address); - TxoutType type_ret; std::vector<CTxDestination> addresses; int required_ret; - (void)ExtractDestinations(script, type_ret, addresses, required_ret); - - const FlatSigningProvider signing_provider; - (void)InferDescriptor(script, signing_provider); - - (void)IsSegWitOutput(signing_provider, script); - - (void)IsSolvable(signing_provider, script); + bool extract_destinations_ret = ExtractDestinations(script, type_ret, addresses, required_ret); + bool extract_destination_ret = ExtractDestination(script, address); + if (!extract_destinations_ret) { + assert(!extract_destination_ret); + if (type_ret == TxoutType::MULTISIG) { + assert(addresses.empty() && required_ret == 0); + } else { + assert(type_ret == TxoutType::PUBKEY || + type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA); + } + } else { + assert(required_ret >= 1 && required_ret <= 16); + assert((unsigned long)required_ret == addresses.size()); + assert(type_ret == TxoutType::MULTISIG || required_ret == 1); + } + if (type_ret == TxoutType::NONSTANDARD || type_ret == TxoutType::NULL_DATA) { + assert(!extract_destinations_ret); + } + if (!extract_destination_ret) { + assert(type_ret == TxoutType::PUBKEY || + type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA || + type_ret == TxoutType::MULTISIG); + } else { + assert(address == addresses[0]); + } + if (type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA || + type_ret == TxoutType::MULTISIG) { + assert(!extract_destination_ret); + } TxoutType which_type; bool is_standard_ret = IsStandard(script, which_type); + assert(type_ret == which_type); if (!is_standard_ret) { assert(which_type == TxoutType::NONSTANDARD || which_type == TxoutType::NULL_DATA || @@ -87,6 +110,11 @@ FUZZ_TARGET_INIT(script, initialize_script) which_type == TxoutType::NONSTANDARD); } + const FlatSigningProvider signing_provider; + (void)InferDescriptor(script, signing_provider); + (void)IsSegWitOutput(signing_provider, script); + (void)IsSolvable(signing_provider, script); + (void)RecursiveDynamicUsage(script); std::vector<std::vector<unsigned char>> solutions; @@ -115,10 +143,12 @@ FUZZ_TARGET_INIT(script, initialize_script) { const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CompressedScript compressed_script; + compressed_script.assign(bytes.begin(), bytes.end()); // DecompressScript(..., ..., bytes) is not guaranteed to be defined if the bytes vector is too short - if (bytes.size() >= 32) { + if (compressed_script.size() >= 32) { CScript decompressed_script; - DecompressScript(decompressed_script, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), bytes); + DecompressScript(decompressed_script, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), compressed_script); } } diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index aa911cdeda..1278dc87d4 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -41,6 +41,10 @@ FUZZ_TARGET_INIT(script_flags, initialize_script_flags) for (unsigned i = 0; i < tx.vin.size(); ++i) { CTxOut prevout; ds >> prevout; + if (!MoneyRange(prevout.nValue)) { + // prevouts should be consensus-valid + prevout.nValue = 1; + } spent_outputs.push_back(prevout); } PrecomputedTransactionData txdata; diff --git a/src/validation.cpp b/src/validation.cpp index 2bf505e26b..1ddafa5613 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4101,38 +4101,46 @@ CVerifyDB::~CVerifyDB() uiInterface.ShowProgress("", 100, false); } -bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_chainstate, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) +bool CVerifyDB::VerifyDB( + CChainState& chainstate, + const CChainParams& chainparams, + CCoinsView& coinsview, + int nCheckLevel, int nCheckDepth) { AssertLockHeld(cs_main); - assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); - if (active_chainstate.m_chain.Tip() == nullptr || active_chainstate.m_chain.Tip()->pprev == nullptr) + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + if (chainstate.m_chain.Tip() == nullptr || chainstate.m_chain.Tip()->pprev == nullptr) return true; // Verify blocks in the best chain - if (nCheckDepth <= 0 || nCheckDepth > active_chainstate.m_chain.Height()) - nCheckDepth = active_chainstate.m_chain.Height(); + if (nCheckDepth <= 0 || nCheckDepth > chainstate.m_chain.Height()) + nCheckDepth = chainstate.m_chain.Height(); nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); - CCoinsViewCache coins(coinsview); + CCoinsViewCache coins(&coinsview); CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; BlockValidationState state; int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ - for (pindex = active_chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { - const int percentageDone = std::max(1, std::min(99, (int)(((double)(active_chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + + bool is_snapshot_cs = !chainstate.m_from_snapshot_blockhash.IsNull(); + + for (pindex = chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { + const int percentageDone = std::max(1, std::min(99, (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone/10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); - if (pindex->nHeight <= active_chainstate.m_chain.Height()-nCheckDepth) + if (pindex->nHeight <= chainstate.m_chain.Height()-nCheckDepth) break; - if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, only go back as far as we have data. + if ((fPruneMode || is_snapshot_cs) && !(pindex->nStatus & BLOCK_HAVE_DATA)) { + // If pruning or running under an assumeutxo snapshot, only go + // back as far as we have data. LogPrintf("VerifyDB(): block verification stopping at height %d (pruning, no data)\n", pindex->nHeight); break; } @@ -4154,9 +4162,11 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_ch } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + active_chainstate.CoinsTip().DynamicMemoryUsage()) <= active_chainstate.m_coinstip_cache_size_bytes) { + size_t curr_coins_usage = coins.DynamicMemoryUsage() + chainstate.CoinsTip().DynamicMemoryUsage(); + + if (nCheckLevel >= 3 && curr_coins_usage <= chainstate.m_coinstip_cache_size_bytes) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); - DisconnectResult res = active_chainstate.DisconnectBlock(block, pindex, coins); + DisconnectResult res = chainstate.DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4170,26 +4180,26 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_ch if (ShutdownRequested()) return true; } if (pindexFailure) - return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", active_chainstate.m_chain.Height() - pindexFailure->nHeight + 1, nGoodTransactions); + return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainstate.m_chain.Height() - pindexFailure->nHeight + 1, nGoodTransactions); // store block count as we move pindex at check level >= 4 - int block_count = active_chainstate.m_chain.Height() - pindex->nHeight; + int block_count = chainstate.m_chain.Height() - pindex->nHeight; // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { - while (pindex != active_chainstate.m_chain.Tip()) { - const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(active_chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); + while (pindex != chainstate.m_chain.Tip()) { + const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); if (reportDone < percentageDone/10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); - pindex = active_chainstate.m_chain.Next(pindex); + pindex = chainstate.m_chain.Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!active_chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) + if (!chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); if (ShutdownRequested()) return true; } @@ -4289,143 +4299,23 @@ bool CChainState::ReplayBlocks(const CChainParams& params) return true; } -//! Helper for CChainState::RewindBlockIndex -void CChainState::EraseBlockData(CBlockIndex* index) +bool CChainState::NeedsRedownload(const CChainParams& params) const { AssertLockHeld(cs_main); - assert(!m_chain.Contains(index)); // Make sure this block isn't active - - // Reduce validity - index->nStatus = std::min<unsigned int>(index->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (index->nStatus & ~BLOCK_VALID_MASK); - // Remove have-data flags. - index->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); - // Remove storage location. - index->nFile = 0; - index->nDataPos = 0; - index->nUndoPos = 0; - // Remove various other things - index->nTx = 0; - index->nChainTx = 0; - index->nSequenceId = 0; - // Make sure it gets written. - setDirtyBlockIndex.insert(index); - // Update indexes - setBlockIndexCandidates.erase(index); - auto ret = m_blockman.m_blocks_unlinked.equal_range(index->pprev); - while (ret.first != ret.second) { - if (ret.first->second == index) { - m_blockman.m_blocks_unlinked.erase(ret.first++); - } else { - ++ret.first; - } - } - // Mark parent as eligible for main chain again - if (index->pprev && index->pprev->IsValid(BLOCK_VALID_TRANSACTIONS) && index->pprev->HaveTxsDownloaded()) { - setBlockIndexCandidates.insert(index->pprev); - } -} - -bool CChainState::RewindBlockIndex(const CChainParams& params) -{ - // Note that during -reindex-chainstate we are called with an empty m_chain! - - // First erase all post-segwit blocks without witness not in the main chain, - // as this can we done without costly DisconnectTip calls. Active - // blocks will be dealt with below (releasing cs_main in between). - { - LOCK(cs_main); - for (const auto& entry : m_blockman.m_block_index) { - if (IsWitnessEnabled(entry.second->pprev, params.GetConsensus()) && !(entry.second->nStatus & BLOCK_OPT_WITNESS) && !m_chain.Contains(entry.second)) { - EraseBlockData(entry.second); - } - } - } - - // Find what height we need to reorganize to. - CBlockIndex *tip; - int nHeight = 1; - { - LOCK(cs_main); - while (nHeight <= m_chain.Height()) { - // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all - // blocks in ConnectBlock, we don't need to go back and - // re-download/re-verify blocks from before segwit actually activated. - if (IsWitnessEnabled(m_chain[nHeight - 1], params.GetConsensus()) && !(m_chain[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { - break; - } - nHeight++; - } - tip = m_chain.Tip(); - } - // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + // At and above params.SegwitHeight, segwit consensus rules must be validated + CBlockIndex* block{m_chain.Tip()}; + const int segwit_height{params.GetConsensus().SegwitHeight}; - BlockValidationState state; - // Loop until the tip is below nHeight, or we reach a pruned block. - while (!ShutdownRequested()) { - { - LOCK(cs_main); - LOCK(m_mempool.cs); - // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) - assert(tip == m_chain.Tip()); - if (tip == nullptr || tip->nHeight < nHeight) break; - if (fPruneMode && !(tip->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, don't try rewinding past the HAVE_DATA point; - // since older blocks can't be served anyway, there's - // no need to walk further, and trying to DisconnectTip() - // will fail (and require a needless reindex/redownload - // of the blockchain). - break; - } - - // Disconnect block - if (!DisconnectTip(state, params, nullptr)) { - return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", tip->nHeight, state.ToString()); - } - - // Reduce validity flag and have-data flags. - // We do this after actual disconnecting, otherwise we'll end up writing the lack of data - // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - // Note: If we encounter an insufficiently validated block that - // is on m_chain, it must be because we are a pruning node, and - // this block or some successor doesn't HAVE_DATA, so we were unable to - // rewind all the way. Blocks remaining on m_chain at this point - // must not have their validity reduced. - EraseBlockData(tip); - - tip = tip->pprev; - } - // Make sure the queue of validation callbacks doesn't grow unboundedly. - LimitValidationInterfaceQueue(); - - // Occasionally flush state to disk. - if (!FlushStateToDisk(params, state, FlushStateMode::PERIODIC)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; - } - } - - { - LOCK(cs_main); - if (m_chain.Tip() != nullptr) { - // We can't prune block index candidates based on our tip if we have - // no tip due to m_chain being empty! - PruneBlockIndexCandidates(); - - CheckBlockIndex(params.GetConsensus()); - - // FlushStateToDisk can possibly read ::ChainActive(). Be conservative - // and skip it here, we're about to -reindex-chainstate anyway, so - // it'll get called a bunch real soon. - BlockValidationState state; - if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; - } + while (block != nullptr && block->nHeight >= segwit_height) { + if (!(block->nStatus & BLOCK_OPT_WITNESS)) { + // block is insufficiently validated for a segwit client + return true; } + block = block->pprev; } - return true; + return false; } void CChainState::UnloadBlockIndex() { diff --git a/src/validation.h b/src/validation.h index de121ab46a..ed1ba4310c 100644 --- a/src/validation.h +++ b/src/validation.h @@ -195,14 +195,14 @@ struct MempoolAcceptResult { VALID, //!> Fully validated, valid. INVALID, //!> Invalid. }; - ResultType m_result_type; - TxValidationState m_state; + const ResultType m_result_type; + const TxValidationState m_state; // The following fields are only present when m_result_type = ResultType::VALID /** Mempool transactions replaced by the tx per BIP 125 rules. */ - std::optional<std::list<CTransactionRef>> m_replaced_transactions; - /** Raw base fees. */ - std::optional<CAmount> m_base_fees; + const std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Raw base fees in satoshis. */ + const std::optional<CAmount> m_base_fees; /** Constructor for failure case */ explicit MempoolAcceptResult(TxValidationState state) @@ -212,7 +212,7 @@ struct MempoolAcceptResult { /** Constructor for success case */ explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) - : m_result_type(ResultType::VALID), m_state(TxValidationState{}), + : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} }; @@ -329,7 +329,12 @@ class CVerifyDB { public: CVerifyDB(); ~CVerifyDB(); - bool VerifyDB(const CChainParams& chainparams, CChainState& active_chainstate, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool VerifyDB( + CChainState& chainstate, + const CChainParams& chainparams, + CCoinsView& coinsview, + int nCheckLevel, + int nCheckDepth) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; enum DisconnectResult @@ -713,7 +718,9 @@ public: /** Replay blocks that aren't fully applied to the database. */ bool ReplayBlocks(const CChainParams& params); - bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); + + /** Whether the chain state needs to be redownloaded due to lack of witness data */ + [[nodiscard]] bool NeedsRedownload(const CChainParams& params) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); @@ -760,9 +767,6 @@ private: bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - //! Mark a block as not having block data - void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void CheckForkWarningConditions() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -914,6 +918,8 @@ public: return m_blockman.m_block_index; } + //! @returns true if a snapshot-based chainstate is in use. Also implies + //! that a background validation chainstate is also in use. bool IsSnapshotActive() const; std::optional<uint256> SnapshotBlockhash() const; diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index 720877ead0..598c5f082c 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -6,21 +6,7 @@ #include <util/system.h> -void CCoinControl::SetNull() +CCoinControl::CCoinControl() { - destChange = CNoDestination(); - m_change_type.reset(); - m_add_inputs = true; - fAllowOtherInputs = false; - fAllowWatchOnly = false; m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS); - m_avoid_address_reuse = false; - setSelected.clear(); - m_feerate.reset(); - fOverrideFeeRate = false; - m_confirm_target.reset(); - m_signal_bip125_rbf.reset(); - m_fee_mode = FeeEstimateMode::UNSET; - m_min_depth = DEFAULT_MIN_DEPTH; - m_max_depth = DEFAULT_MAX_DEPTH; } diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index d25a3fb3fa..716e1922fe 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -24,17 +24,17 @@ class CCoinControl { public: //! Custom change destination, if not set an address is generated - CTxDestination destChange; + CTxDestination destChange = CNoDestination(); //! Override the default change type if set, ignored if destChange is set std::optional<OutputType> m_change_type; //! If false, only selected inputs are used - bool m_add_inputs; + bool m_add_inputs = true; //! If false, allows unselected inputs, but requires all selected inputs be used - bool fAllowOtherInputs; + bool fAllowOtherInputs = false; //! Includes watch only addresses which are solvable - bool fAllowWatchOnly; + bool fAllowWatchOnly = false; //! Override automatic min/max checks on fee, m_feerate must be set if true - bool fOverrideFeeRate; + bool fOverrideFeeRate = false; //! Override the wallet's m_pay_tx_fee if set std::optional<CFeeRate> m_feerate; //! Override the default confirmation target if set @@ -42,22 +42,17 @@ public: //! Override the wallet's m_signal_rbf if set std::optional<bool> m_signal_bip125_rbf; //! Avoid partial use of funds sent to a given address - bool m_avoid_partial_spends; + bool m_avoid_partial_spends = DEFAULT_AVOIDPARTIALSPENDS; //! Forbids inclusion of dirty (previously used) addresses - bool m_avoid_address_reuse; + bool m_avoid_address_reuse = false; //! Fee estimation mode to control arguments to estimateSmartFee - FeeEstimateMode m_fee_mode; + FeeEstimateMode m_fee_mode = FeeEstimateMode::UNSET; //! Minimum chain depth value for coin availability int m_min_depth = DEFAULT_MIN_DEPTH; //! Maximum chain depth value for coin availability int m_max_depth = DEFAULT_MAX_DEPTH; - CCoinControl() - { - SetNull(); - } - - void SetNull(); + CCoinControl(); bool HasSelected() const { diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 4e2de1daf4..6fc8773ee3 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -166,6 +166,8 @@ class NotificationsTest(BitcoinTestFramework): # Should now verify contents of each file for tx_id, blockheight, blockhash in tx_details: fname = os.path.join(self.walletnotify_dir, notify_outputname(self.wallet, tx_id)) + # Wait for the cached writes to hit storage + self.wait_until(lambda: os.path.getsize(fname) > 0, timeout=10) with open(fname, 'rt', encoding='utf-8') as f: text = f.read() # Universal newline ensures '\n' on 'nt' diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 69821763bd..87297989ba 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -11,6 +11,7 @@ from test_framework.messages import ( NODE_NETWORK, NODE_WITNESS, msg_addr, + msg_getaddr ) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework @@ -19,18 +20,6 @@ from test_framework.util import ( ) import time -# Keep this with length <= 10. Addresses from larger messages are not relayed. -ADDRS = [] -num_ipv4_addrs = 10 - -for i in range(num_ipv4_addrs): - addr = CAddress() - addr.time = int(time.time()) + i - addr.nServices = NODE_NETWORK | NODE_WITNESS - addr.ip = "123.123.123.{}".format(i % 256) - addr.port = 8333 + i - ADDRS.append(addr) - class AddrReceiver(P2PInterface): num_ipv4_received = 0 @@ -44,36 +33,87 @@ class AddrReceiver(P2PInterface): self.num_ipv4_received += 1 +class GetAddrStore(P2PInterface): + getaddr_received = False + num_ipv4_received = 0 + + def on_getaddr(self, message): + self.getaddr_received = True + + def on_addr(self, message): + for addr in message.addrs: + self.num_ipv4_received += 1 + + def addr_received(self): + return self.num_ipv4_received != 0 + + class AddrTest(BitcoinTestFramework): + counter = 0 + mocktime = int(time.time()) + def set_test_params(self): self.num_nodes = 1 def run_test(self): - self.log.info('Create connection that sends addr messages') - addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + self.oversized_addr_test() + self.relay_tests() + self.getaddr_tests() + self.blocksonly_mode_tests() + + def setup_addr_msg(self, num): + addrs = [] + for i in range(num): + addr = CAddress() + addr.time = self.mocktime + i + addr.nServices = NODE_NETWORK | NODE_WITNESS + addr.ip = f"123.123.123.{self.counter % 256}" + addr.port = 8333 + i + addrs.append(addr) + self.counter += 1 + msg = msg_addr() + msg.addrs = addrs + return msg + + def send_addr_msg(self, source, msg, receivers): + source.send_and_ping(msg) + # pop m_next_addr_send timer + self.mocktime += 5 * 60 + self.nodes[0].setmocktime(self.mocktime) + for peer in receivers: + peer.sync_send_with_ping() - self.log.info('Send too-large addr message') - msg.addrs = ADDRS * 101 # more than 1000 addresses in one message + def oversized_addr_test(self): + self.log.info('Send an addr message that is too large') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + + msg = self.setup_addr_msg(1010) with self.nodes[0].assert_debug_log(['addr message size = 1010']): addr_source.send_and_ping(msg) + self.nodes[0].disconnect_p2ps() + + def relay_tests(self): + self.log.info('Test address relay') self.log.info('Check that addr message content is relayed and added to addrman') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) num_receivers = 7 receivers = [] for _ in range(num_receivers): receivers.append(self.nodes[0].add_p2p_connection(AddrReceiver())) - msg.addrs = ADDRS + + # Keep this with length <= 10. Addresses from larger messages are not + # relayed. + num_ipv4_addrs = 10 + msg = self.setup_addr_msg(num_ipv4_addrs) with self.nodes[0].assert_debug_log( [ 'Added {} addresses from 127.0.0.1: 0 tried'.format(num_ipv4_addrs), - 'received: addr (301 bytes) peer=0', + 'received: addr (301 bytes) peer=1', ] ): - addr_source.send_and_ping(msg) - self.nodes[0].setmocktime(int(time.time()) + 30 * 60) - for receiver in receivers: - receiver.sync_with_ping() + self.send_addr_msg(addr_source, msg, receivers) total_ipv4_received = sum(r.num_ipv4_received for r in receivers) @@ -82,6 +122,92 @@ class AddrTest(BitcoinTestFramework): ipv4_branching_factor = 2 assert_equal(total_ipv4_received, num_ipv4_addrs * ipv4_branching_factor) + self.nodes[0].disconnect_p2ps() + + self.log.info('Check relay of addresses received from outbound peers') + inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=0, connection_type="outbound-full-relay") + msg = self.setup_addr_msg(2) + self.send_addr_msg(full_outbound_peer, msg, [inbound_peer]) + self.log.info('Check that the first addr message received from an outbound peer is not relayed') + # Currently, there is a flag that prevents the first addr message received + # from a new outbound peer to be relayed to others. Originally meant to prevent + # large GETADDR responses from being relayed, it now typically affects the self-announcement + # of the outbound peer which is often sent before the GETADDR response. + assert_equal(inbound_peer.num_ipv4_received, 0) + + self.log.info('Check that subsequent addr messages sent from an outbound peer are relayed') + msg2 = self.setup_addr_msg(2) + self.send_addr_msg(full_outbound_peer, msg2, [inbound_peer]) + assert_equal(inbound_peer.num_ipv4_received, 2) + + self.log.info('Check address relay to outbound peers') + block_relay_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=1, connection_type="block-relay-only") + msg3 = self.setup_addr_msg(2) + self.send_addr_msg(inbound_peer, msg3, [full_outbound_peer, block_relay_peer]) + + self.log.info('Check that addresses are relayed to full outbound peers') + assert_equal(full_outbound_peer.num_ipv4_received, 2) + self.log.info('Check that addresses are not relayed to block-relay-only outbound peers') + assert_equal(block_relay_peer.num_ipv4_received, 0) + + self.nodes[0].disconnect_p2ps() + + def getaddr_tests(self): + self.log.info('Test getaddr behavior') + self.log.info('Check that we send a getaddr message upon connecting to an outbound-full-relay peer') + full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=0, connection_type="outbound-full-relay") + full_outbound_peer.sync_with_ping() + assert full_outbound_peer.getaddr_received + + self.log.info('Check that we do not send a getaddr message upon connecting to a block-relay-only peer') + block_relay_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=1, connection_type="block-relay-only") + block_relay_peer.sync_with_ping() + assert_equal(block_relay_peer.getaddr_received, False) + + self.log.info('Check that we answer getaddr messages only from inbound peers') + inbound_peer = self.nodes[0].add_p2p_connection(GetAddrStore()) + inbound_peer.sync_with_ping() + + # Add some addresses to addrman + for i in range(1000): + first_octet = i >> 8 + second_octet = i % 256 + a = f"{first_octet}.{second_octet}.1.1" + self.nodes[0].addpeeraddress(a, 8333) + + full_outbound_peer.send_and_ping(msg_getaddr()) + block_relay_peer.send_and_ping(msg_getaddr()) + inbound_peer.send_and_ping(msg_getaddr()) + + self.mocktime += 5 * 60 + self.nodes[0].setmocktime(self.mocktime) + inbound_peer.wait_until(inbound_peer.addr_received) + + assert_equal(full_outbound_peer.num_ipv4_received, 0) + assert_equal(block_relay_peer.num_ipv4_received, 0) + assert inbound_peer.num_ipv4_received > 100 + + self.nodes[0].disconnect_p2ps() + + def blocksonly_mode_tests(self): + self.log.info('Test addr relay in -blocksonly mode') + self.restart_node(0, ["-blocksonly"]) + self.mocktime = int(time.time()) + + self.log.info('Check that we send getaddr messages') + full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=0, connection_type="outbound-full-relay") + full_outbound_peer.sync_with_ping() + assert full_outbound_peer.getaddr_received + + self.log.info('Check that we relay address messages') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + msg = self.setup_addr_msg(2) + self.send_addr_msg(addr_source, msg, [full_outbound_peer]) + assert_equal(full_outbound_peer.num_ipv4_received, 2) + + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6584efae79..445cea6186 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -89,11 +89,7 @@ class P2PBlocksOnly(BitcoinTestFramework): # Bump time forward to ensure nNextInvSend timer pops self.nodes[0].setmocktime(int(time.time()) + 60) - # Calling sync_with_ping twice requires that the node calls - # `ProcessMessage` twice, and thus ensures `SendMessages` must have - # been called at least once - conn.sync_with_ping() - conn.sync_with_ping() + conn.sync_send_with_ping() assert(int(txid, 16) not in conn.get_invs()) def check_p2p_tx_violation(self, index=1): diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 4bee33f825..359cfb9c34 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -174,8 +174,7 @@ class FilterTest(BitcoinTestFramework): filter_peer.merkleblock_received = False filter_peer.tx_received = False self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 90) - filter_peer.sync_with_ping() - filter_peer.sync_with_ping() + filter_peer.sync_send_with_ping() assert not filter_peer.merkleblock_received assert not filter_peer.tx_received diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 54891b07e1..14a4afc2d7 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1956,22 +1956,33 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - self.restart_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) + # All nodes are caught up and node 2 is a pre-segwit node that will soon upgrade. + for n in range(2): + assert_equal(self.nodes[n].getblockcount(), self.nodes[2].getblockcount()) + assert softfork_active(self.nodes[n], "segwit") + assert SEGWIT_HEIGHT < self.nodes[2].getblockcount() + assert 'segwit' not in self.nodes[2].getblockchaininfo()['softforks'] + + # Restarting node 2 should result in a shutdown because the blockchain consists of + # insufficiently validated blocks per segwit consensus rules. + self.stop_node(2) + with self.nodes[2].assert_debug_log(expected_msgs=[ + f"Witness data for blocks after height {SEGWIT_HEIGHT} requires validation. Please restart with -reindex."], timeout=10): + self.nodes[2].start([f"-segwitheight={SEGWIT_HEIGHT}"]) + + # As directed, the user restarts the node with -reindex + self.start_node(2, extra_args=["-reindex", f"-segwitheight={SEGWIT_HEIGHT}"]) + + # With the segwit consensus rules, the node is able to validate only up to SEGWIT_HEIGHT - 1 + assert_equal(self.nodes[2].getblockcount(), SEGWIT_HEIGHT - 1) self.connect_nodes(0, 2) # We reconnect more than 100 blocks, give it plenty of time + # sync_blocks() also verifies the best block hash is the same for all nodes self.sync_blocks(timeout=240) - # Make sure that this peer thinks segwit has activated. - assert softfork_active(self.nodes[2], 'segwit') - - # Make sure this peer's blocks match those of node0. - height = self.nodes[2].getblockcount() - while height >= 0: - block_hash = self.nodes[2].getblockhash(height) - assert_equal(block_hash, self.nodes[0].getblockhash(height)) - assert_equal(self.nodes[0].getblock(block_hash), self.nodes[2].getblock(block_hash)) - height -= 1 + # The upgraded node should now have segwit activated + assert softfork_active(self.nodes[2], "segwit") @subtest # type: ignore def test_witness_sigops(self): diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 1398d1237f..a80fa596cd 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -61,6 +61,9 @@ class RpcMiscTest(BitcoinTestFramework): node.logging(include=['qt']) assert_equal(node.logging()['qt'], True) + self.log.info("test echoipc (testing spawned process in multiprocess build)") + assert_equal(node.echoipc("hello"), "hello") + self.log.info("test getindexinfo") # Without any indices running the RPC returns an empty object assert_equal(node.getindexinfo(), {}) diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 05099f3339..cc80b543cd 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -539,8 +539,16 @@ class P2PInterface(P2PConnection): self.send_message(message) self.sync_with_ping(timeout=timeout) - # Sync up with the node + def sync_send_with_ping(self, timeout=60): + """Ensure SendMessages is called on this connection""" + # Calling sync_with_ping twice requires that the node calls + # `ProcessMessage` twice, and thus ensures `SendMessages` must have + # been called at least once + self.sync_with_ping() + self.sync_with_ping() + def sync_with_ping(self, timeout=60): + """Ensure ProcessMessages is called on this connection""" self.send_message(msg_ping(nonce=self.ping_counter)) def test_function(): diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index 5cfa41537f..c23b903bce 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -15,7 +15,7 @@ REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/ EXIT_CODE=0 for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr "[:lower:]" "[:upper:]") + HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then echo "${HEADER_FILE} seems to be missing the expected include guard:" |