diff options
114 files changed, 2544 insertions, 505 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f9b47ff7d9..1a972e1e4b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -171,7 +171,7 @@ task: - cd %CIRRUS_WORKING_DIR% - ccache --zero-stats --max-size=%CCACHE_SIZE% - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL% -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - ccache --show-stats check_script: - src\test_bitcoin.exe -l test_suite @@ -184,7 +184,7 @@ task: - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - netsh int ipv6 set dynamicport tcp start=1025 num=64511 # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash task: name: 'ARM [unit tests, no functional tests] [bullseye]' diff --git a/.tx/config b/.tx/config index 37c0bca0ab..20eff98d28 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[bitcoin.qt-translation-024x] +[o:bitcoin:p:bitcoin:r:qt-translation-024x] file_filter = src/qt/locale/bitcoin_<lang>.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index ec572b4f2e..3a2540d549 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -54,6 +54,9 @@ <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj"> <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project> </ProjectReference> + <ProjectReference Include="..\libtest_util\libtest_util.vcxproj"> + <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project> + </ProjectReference> <ProjectReference Include="..\libleveldb\libleveldb.vcxproj"> <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> </ProjectReference> diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 5bdb392ba3..0ee80cf114 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -68,6 +68,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/util/string.cpp"\ " src/util/syserror.cpp"\ " src/util/url.cpp"\ + " src/zmq"\ " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi diff --git a/configure.ac b/configure.ac index 936ade6bc3..f2d621ec7f 100644 --- a/configure.ac +++ b/configure.ac @@ -763,10 +763,6 @@ case $host in BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi - if test "$use_sqlite" != "no" && $BREW list --versions sqlite3 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" - fi - if $BREW list --versions qt@5 >/dev/null; then export PKG_CONFIG_PATH="$($BREW --prefix qt@5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index f8377cce33..2d96b863ef 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -1,3 +1,4 @@ +982A193E3CE0EED535E09023188CBB2648416AD5 0xB10C (0xb10c) 9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) 617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 99f5b0a8db..7ad2529e47 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -3,17 +3,20 @@ $(package)_version=2.2.2 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=888fb0976ba61518276fe1eda988589c700a3f2a69d71089260d75562afd3687 -$(package)_patches=dont_leak_info.patch +$(package)_patches=dont_leak_info.patch respect_mingw_cflags.patch +# Next time this package is updated, ensure that _WIN32_WINNT is still properly set. +# See discussion in https://github.com/bitcoin/bitcoin/pull/25964. define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" $(package)_build_opts_darwin=LIBTOOL="$($(package)_libtool)" -$(package)_build_opts_mingw32=-f Makefile.mingw +$(package)_build_opts_mingw32=-f Makefile.mingw CFLAGS="$($(package)_cflags) -D_WIN32_WINNT=0x0601" $(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch + patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch && \ + patch -p1 < $($(package)_patch_dir)/respect_mingw_cflags.patch endef define $(package)_build_cmds diff --git a/depends/patches/miniupnpc/respect_mingw_cflags.patch b/depends/patches/miniupnpc/respect_mingw_cflags.patch new file mode 100644 index 0000000000..a44580ddab --- /dev/null +++ b/depends/patches/miniupnpc/respect_mingw_cflags.patch @@ -0,0 +1,23 @@ +commit fec515a7ac9991a0ee91068fda046b54b191155e +Author: fanquake <fanquake@gmail.com> +Date: Wed Jul 27 15:52:37 2022 +0100 + + build: respect CFLAGS in makefile.mingw + + Similar to the other Makefile. + + Cherry-pick of https://github.com/miniupnp/miniupnp/pull/619. + +diff --git a/Makefile.mingw b/Makefile.mingw +index 2bff7bd..88430d2 100644 +--- a/Makefile.mingw ++++ b/Makefile.mingw +@@ -19,7 +19,7 @@ else + RM = rm -f + endif + #CFLAGS = -Wall -g -DDEBUG -D_WIN32_WINNT=0X501 +-CFLAGS = -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 ++CFLAGS ?= -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 + LDLIBS = -lws2_32 -liphlpapi + # -lwsock32 + # -liphlpapi is needed for GetBestRoute() and GetIpAddrTable() diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 265b74ee9a..a704b969df 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -37,8 +37,8 @@ By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks -`GET /rest/block/<BLOCK-HASH>.<bin|hex|json>` -`GET /rest/block/notxdetails/<BLOCK-HASH>.<bin|hex|json>` +- `GET /rest/block/<BLOCK-HASH>.<bin|hex|json>` +- `GET /rest/block/notxdetails/<BLOCK-HASH>.<bin|hex|json>` Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. @@ -86,12 +86,23 @@ Returns various state info regarding block chain processing. Only supports JSON as output format. Refer to the `getblockchaininfo` RPC help for details. +#### Deployment info +`GET /rest/deploymentinfo.json` +`GET /rest/deploymentinfo/<BLOCKHASH>.json` + +Returns an object containing various state info regarding deployments of +consensus changes at the current chain tip, or at <BLOCKHASH> if provided. +Only supports JSON as output format. +Refer to the `getdeploymentinfo` RPC help for details. + #### Query UTXO set -`GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n>/.../<txid>-<n>.<bin|hex|json>` +- `GET /rest/getutxos/<TXID>-<N>/<TXID>-<N>/.../<TXID>-<N>.<bin|hex|json>` +- `GET /rest/getutxos/checkmempool/<TXID>-<N>/<TXID>-<N>/.../<TXID>-<N>.<bin|hex|json>` -The getutxo command allows querying of the UTXO set given a set of outpoints. -See BIP64 for input and output serialisation: -https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki +The getutxos endpoint allows querying the UTXO set, given a set of outpoints. +With the `/checkmempool/` option, the mempool is also taken into account. +See [BIP64](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) for +input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index c353c78ff8..ea51b1b87f 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -76,8 +76,9 @@ original chainstate remains in use as active. Once the snapshot chainstate is loaded and validated, it is promoted to active chainstate and a sync to tip begins. A new chainstate directory is created in the -datadir for the snapshot chainstate called -`chainstate_[SHA256 blockhash of snapshot base block]`. +datadir for the snapshot chainstate called `chainstate_snapshot`. When this directory +is present in the datadir, the snapshot chainstate will be detected and loaded as +active on node startup (via `DetectSnapshotChainstate()`). | | | | ---------- | ----------- | diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 75f8d60ba0..5e51f1e63f 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -83,7 +83,7 @@ class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold </td></tr></table> -- The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code still is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. +- The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. - *libbitcoin_consensus* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. diff --git a/doc/release-notes-22087.md b/doc/release-notes-22087.md new file mode 100644 index 0000000000..8d7fd242b2 --- /dev/null +++ b/doc/release-notes-22087.md @@ -0,0 +1,4 @@ +Updated settings +---------------- + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. Values that previously worked and were considered valid can now result in errors. (#22087) diff --git a/doc/release-notes-25412.md b/doc/release-notes-25412.md new file mode 100644 index 0000000000..b11fe73d45 --- /dev/null +++ b/doc/release-notes-25412.md @@ -0,0 +1,5 @@ +New REST endpoint +----------------- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) diff --git a/src/Makefile.am b/src/Makefile.am index caeaee511f..70a0ca8915 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -209,6 +209,7 @@ BITCOIN_CORE_H = \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ + node/txreconciliation.h \ node/utxo_snapshot.h \ node/validation_cache_args.h \ noui.h \ @@ -345,11 +346,7 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h - -# server: shared between bitcoind and bitcoin-qt -# Contains code accessing mempool and chain state that is meant to be separated -# from wallet and gui code (see node/README.md). Shared code should go in -# libbitcoin_common or libbitcoin_util libraries, instead. +# node # libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ @@ -396,6 +393,8 @@ libbitcoin_node_a_SOURCES = \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ + node/txreconciliation.cpp \ + node/utxo_snapshot.cpp \ node/validation_cache_args.cpp \ noui.cpp \ policy/fees.cpp \ @@ -438,7 +437,9 @@ endif if !ENABLE_WALLET libbitcoin_node_a_SOURCES += dummywallet.cpp endif +# +# zmq # if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -449,10 +450,9 @@ libbitcoin_zmq_a_SOURCES = \ zmq/zmqrpc.cpp \ zmq/zmqutil.cpp endif +# - -# wallet: shared between bitcoind and bitcoin-qt, but only linked -# when wallet enabled +# wallet # libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(BDB_CPPFLAGS) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ @@ -491,14 +491,17 @@ endif if USE_BDB libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp endif +# +# wallet tool # libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_tool_a_SOURCES = \ wallet/wallettool.cpp \ $(BITCOIN_CORE_H) +# -# crypto primitives library +# crypto # crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) # Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a @@ -578,8 +581,9 @@ crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp +# -# consensus: shared between all executables that validate any consensus rules. +# consensus # libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ @@ -615,8 +619,9 @@ libbitcoin_consensus_a_SOURCES = \ util/strencodings.cpp \ util/strencodings.h \ version.h +# -# common: shared between bitcoind, and bitcoin-qt and non-server tools +# common # libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ @@ -655,8 +660,9 @@ libbitcoin_common_a_SOURCES = \ script/standard.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) +# -# util: shared between all executables. +# util # libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ @@ -704,8 +710,9 @@ libbitcoin_util_a_SOURCES = \ if USE_LIBEVENT libbitcoin_util_a_SOURCES += util/url.cpp endif +# -# cli: shared between bitcoin-cli and bitcoin-qt +# cli # libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ @@ -902,6 +909,7 @@ libbitcoinkernel_la_SOURCES = \ node/blockstorage.cpp \ node/chainstate.cpp \ node/interface_ui.cpp \ + node/utxo_snapshot.cpp \ policy/feerate.cpp \ policy/fees.cpp \ policy/packages.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 253f64d2c3..fc60359efd 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -77,6 +77,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -147,6 +148,7 @@ BITCOIN_TESTS =\ test/transaction_tests.cpp \ test/txindex_tests.cpp \ test/txpackage_tests.cpp \ + test/txreconciliation_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index ada789f1b0..d142572b27 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -5,20 +5,21 @@ LIBTEST_UTIL=libtest_util.a EXTRA_LIBRARIES += \ - $(LIBTEST_UTIL) + $(LIBTEST_UTIL) TEST_UTIL_H = \ - test/util/blockfilter.h \ - test/util/chainstate.h \ - test/util/logging.h \ - test/util/mining.h \ - test/util/net.h \ - test/util/script.h \ - test/util/setup_common.h \ - test/util/str.h \ - test/util/transaction_utils.h \ - test/util/validation.h \ - test/util/wallet.h + test/util/blockfilter.h \ + test/util/chainstate.h \ + test/util/logging.h \ + test/util/mining.h \ + test/util/net.h \ + test/util/script.h \ + test/util/setup_common.h \ + test/util/str.h \ + test/util/transaction_utils.h \ + test/util/txmempool.h \ + test/util/validation.h \ + test/util/wallet.h libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -31,6 +32,7 @@ libtest_util_a_SOURCES = \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ + test/util/txmempool.cpp \ test/util/validation.cpp \ test/util/wallet.cpp \ $(TEST_UTIL_H) diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index 5c868a8573..615e53d337 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <key.h> +#include <pubkey.h> #include <script/descriptor.h> #include <script/standard.h> @@ -12,6 +13,9 @@ static void ExpandDescriptor(benchmark::Bench& bench) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; const std::pair<int64_t, int64_t> range = {0, 1000}; FlatSigningProvider provider; @@ -25,6 +29,8 @@ static void ExpandDescriptor(benchmark::Bench& bench) assert(success); } }); + + ECC_Stop(); } BENCHMARK(ExpandDescriptor); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index e64e2202ba..6d77385584 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -642,7 +642,7 @@ public: " send Time since last message sent to the peer, in seconds\n" " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" - " \"*\" - the peer requested we not relay transactions to it (relaytxes is false)\n" + " \"*\" - whether we relay transactions to this peer (relaytxes is false)\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" " hb High-bandwidth BIP152 compact block relay\n" " \".\" (to) - we selected the peer as a high-bandwidth peer\n" diff --git a/src/compat/compat.h b/src/compat/compat.h index a8e5552c0a..cc37797577 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -109,14 +109,6 @@ typedef char* sockopt_arg_type; #define USE_POLL #endif -bool static inline IsSelectableSocket(const SOCKET& s) { -#if defined(USE_POLL) || defined(WIN32) - return true; -#else - return (s < FD_SETSIZE); -#endif -} - // MSG_NOSIGNAL is not available on some platforms, if it doesn't exist define it as 0 #if !defined(MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 4dbc839941..7f45e35aef 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -128,7 +128,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) } CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) - : m_name{fs::PathToString(path.stem())} + : m_name{fs::PathToString(path.stem())}, m_path{path}, m_is_memory{fMemory} { penv = nullptr; readoptions.verify_checksums = true; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 665eaa0e98..1052da01d5 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -39,6 +39,10 @@ public: class CDBWrapper; +namespace dbwrapper { + using leveldb::DestroyDB; +} + /** These should be considered an implementation detail of the specific database. */ namespace dbwrapper_private { @@ -219,6 +223,12 @@ private: std::vector<unsigned char> CreateObfuscateKey() const; + //! path to filesystem storage + const fs::path m_path; + + //! whether or not the database resides in memory + bool m_is_memory; + public: /** * @param[in] path Location in the filesystem where leveldb data will be stored. @@ -268,6 +278,14 @@ public: return WriteBatch(batch, fSync); } + //! @returns filesystem path to the on-disk data. + std::optional<fs::path> StoragePath() { + if (m_is_memory) { + return {}; + } + return m_path; + } + template <typename K> bool Exists(const K& key) const { diff --git a/src/init.cpp b/src/init.cpp index 25b40c6c6e..cf4102e6a7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -45,6 +45,7 @@ #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> +#include <node/txreconciliation.h> #include <node/validation_cache_args.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -484,6 +485,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); @@ -1255,6 +1257,51 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // as they would never get updated. if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(args)); + // Check port numbers + for (const std::string port_option : { + "-port", + "-rpcport", + }) { + if (args.IsArgSet(port_option)) { + const std::string port = args.GetArg(port_option, ""); + uint16_t n; + if (!ParseUInt16(port, &n) || n == 0) { + return InitError(InvalidPortErrMsg(port_option, port)); + } + } + } + + for (const std::string port_option : { + "-i2psam", + "-onion", + "-proxy", + "-rpcbind", + "-torcontrol", + "-whitebind", + "-zmqpubhashblock", + "-zmqpubhashtx", + "-zmqpubrawblock", + "-zmqpubrawtx", + "-zmqpubsequence", + }) { + for (const std::string& socket_addr : args.GetArgs(port_option)) { + std::string host_out; + uint16_t port_out{0}; + if (!SplitHostPort(socket_addr, port_out, host_out)) { + return InitError(InvalidPortErrMsg(port_option, socket_addr)); + } + } + } + + for (const std::string& socket_addr : args.GetArgs("-bind")) { + std::string host_out; + uint16_t port_out{0}; + std::string bind_socket_addr = socket_addr.substr(0, socket_addr.rfind('=')); + if (!SplitHostPort(bind_socket_addr, port_out, host_out)) { + return InitError(InvalidPortErrMsg("-bind", socket_addr)); + } + } + // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : args.GetArgs("-uacomment")) { @@ -1663,7 +1710,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const auto BadPortWarning = [](const char* prefix, uint16_t port) { return strprintf(_("%s request to listen on port %u. This port is considered \"bad\" and " - "thus it is unlikely that any Bitcoin Core peers connect to it. See " + "thus it is unlikely that any peer will connect to it. See " "doc/p2p-bad-ports.md for details and a full list."), prefix, port); diff --git a/src/logging.cpp b/src/logging.cpp index a3f1d39be5..c95c0b7e37 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -180,6 +180,7 @@ const CLogCategoryDesc LogCategories[] = #endif {BCLog::UTIL, "util"}, {BCLog::BLOCKSTORE, "blockstorage"}, + {BCLog::TXRECONCILIATION, "txreconciliation"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -280,6 +281,8 @@ std::string LogCategoryToStr(BCLog::LogFlags category) return "util"; case BCLog::LogFlags::BLOCKSTORE: return "blockstorage"; + case BCLog::LogFlags::TXRECONCILIATION: + return "txreconciliation"; case BCLog::LogFlags::ALL: return "all"; } diff --git a/src/logging.h b/src/logging.h index fe91ee43a5..5ee6665c76 100644 --- a/src/logging.h +++ b/src/logging.h @@ -66,6 +66,7 @@ namespace BCLog { #endif UTIL = (1 << 25), BLOCKSTORE = (1 << 26), + TXRECONCILIATION = (1 << 27), ALL = ~(uint32_t)0, }; enum class Level { diff --git a/src/net.cpp b/src/net.cpp index 0736f3ef1b..3c28b9eddf 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -970,8 +970,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, return; } - if (!IsSelectableSocket(sock->Get())) - { + if (!sock->IsSelectable()) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); return; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index eca6263392..c87b0e7cd2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -20,6 +20,7 @@ #include <netbase.h> #include <netmessagemaker.h> #include <node/blockstorage.h> +#include <node/txreconciliation.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -271,12 +272,7 @@ struct Peer { struct TxRelay { mutable RecursiveMutex m_bloom_filter_mutex; - /** Whether the peer wishes to receive transaction announcements. - * - * This is initially set based on the fRelay flag in the received - * `version` message. If initially set to false, it can only be flipped - * to true if we have offered the peer NODE_BLOOM services and it sends - * us a `filterload` or `filterclear` message. See BIP37. */ + /** Whether we relay transactions to this peer. */ bool m_relay_txs GUARDED_BY(m_bloom_filter_mutex){false}; /** A bloom filter for which transactions to announce to the peer. See BIP37. */ std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; @@ -708,6 +704,7 @@ private: ChainstateManager& m_chainman; CTxMemPool& m_mempool; TxRequestTracker m_txrequest GUARDED_BY(::cs_main); + std::unique_ptr<TxReconciliationTracker> m_txreconciliation; /** The height of the best chain */ std::atomic<int> m_best_height{-1}; @@ -1497,6 +1494,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) } WITH_LOCK(g_cs_orphans, m_orphanage.EraseForPeer(nodeid)); m_txrequest.DisconnectedPeer(nodeid); + if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; m_peers_downloading_from -= (state->nBlocksInFlight != 0); assert(m_peers_downloading_from >= 0); @@ -1781,6 +1779,11 @@ PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, m_mempool(pool), m_ignore_incoming_txs(ignore_incoming_txs) { + // While Erlay support is incomplete, it must be enabled explicitly via -txreconciliation. + // This argument can go away after Erlay support is complete. + if (gArgs.GetBoolArg("-txreconciliation", DEFAULT_TXRECONCILIATION_ENABLE)) { + m_txreconciliation = std::make_unique<TxReconciliationTracker>(TXRECONCILIATION_VERSION); + } } void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) @@ -3241,8 +3244,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); } - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); - pfrom.m_has_all_wanted_services = HasAllDesirableServiceFlags(nServices); peer->m_their_services = nServices; pfrom.SetAddrLocal(addrMe); @@ -3252,10 +3253,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } peer->m_starting_height = starting_height; - // We only initialize the m_tx_relay data structure if: - // - this isn't an outbound block-relay-only connection; and - // - fRelay=true or we're offering NODE_BLOOM to this peer - // (NODE_BLOOM means that the peer may turn on tx relay later) + // We only initialize the Peer::TxRelay m_relay_txs data structure if: + // - this isn't an outbound block-relay-only connection, and + // - fRelay=true (the peer wishes to receive transaction announcements) + // or we're offering NODE_BLOOM to this peer. NODE_BLOOM means that + // the peer may turn on transaction relay later. if (!pfrom.IsBlockOnlyConn() && (fRelay || (peer->m_our_services & NODE_BLOOM))) { auto* const tx_relay = peer->SetTxRelay(); @@ -3266,6 +3268,25 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fRelay) pfrom.m_relays_txs = true; } + if (greatest_common_version >= WTXID_RELAY_VERSION && m_txreconciliation) { + // Per BIP-330, we announce txreconciliation support if: + // - protocol version per the VERSION message supports WTXID_RELAY; + // - we intended to exchange transactions over this connection while establishing it + // and the peer indicated support for transaction relay in the VERSION message; + // - we are not in -blocksonly mode. + if (pfrom.m_relays_txs && !m_ignore_incoming_txs) { + const uint64_t recon_salt = m_txreconciliation->PreRegisterPeer(pfrom.GetId()); + // We suggest our txreconciliation role (initiator/responder) based on + // the connection direction. + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDTXRCNCL, + !pfrom.IsInboundConn(), + pfrom.IsInboundConn(), + TXRECONCILIATION_VERSION, recon_salt)); + } + } + + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); + // Potentially mark this peer as a preferred download peer. { LOCK(cs_main); @@ -3393,6 +3414,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // they may wish to request compact blocks from us m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION)); } + + if (m_txreconciliation) { + if (!peer->m_wtxid_relay || !m_txreconciliation->IsPeerRegistered(pfrom.GetId())) { + // We could have optimistically pre-registered/registered the peer. In that case, + // we should forget about the reconciliation state here if this wasn't followed + // by WTXIDRELAY (since WTXIDRELAY can't be announced later). + m_txreconciliation->ForgetPeer(pfrom.GetId()); + } + } + pfrom.fSuccessfullyConnected = true; return; } @@ -3456,6 +3487,58 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Received from a peer demonstrating readiness to announce transactions via reconciliations. + // This feature negotiation must happen between VERSION and VERACK to avoid relay problems + // from switching announcement protocols after the connection is up. + if (msg_type == NetMsgType::SENDTXRCNCL) { + if (!m_txreconciliation) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl from peer=%d ignored, as our node does not have txreconciliation enabled\n", pfrom.GetId()); + return; + } + + if (pfrom.fSuccessfullyConnected) { + // Disconnect peers that send a SENDTXRCNCL message after VERACK. + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received after verack from peer=%d; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + if (!peer->GetTxRelay()) { + // Disconnect peers that send a SENDTXRCNCL message even though we indicated we don't + // support transaction relay. + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received from peer=%d to which we indicated no tx relay; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + bool is_peer_initiator, is_peer_responder; + uint32_t peer_txreconcl_version; + uint64_t remote_salt; + vRecv >> is_peer_initiator >> is_peer_responder >> peer_txreconcl_version >> remote_salt; + + if (m_txreconciliation->IsPeerRegistered(pfrom.GetId())) { + // A peer is already registered, meaning we already received SENDTXRCNCL from them. + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d (sendtxrcncl received from already registered peer); disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + const ReconciliationRegisterResult result = m_txreconciliation->RegisterPeer(pfrom.GetId(), pfrom.IsInboundConn(), + is_peer_initiator, is_peer_responder, + peer_txreconcl_version, + remote_salt); + + // If it's a protocol violation, disconnect. + // If the peer was not found (but something unexpected happened) or it was registered, + // nothing to be done. + if (result == ReconciliationRegisterResult::PROTOCOL_VIOLATION) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + return; + } + if (!pfrom.fSuccessfullyConnected) { LogPrint(BCLog::NET, "Unsupported message \"%s\" prior to verack from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return; diff --git a/src/netbase.cpp b/src/netbase.cpp index a5f7bda875..8169b40ea6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -304,8 +304,7 @@ enum class IntrRecvError { * read. * * @see This function can be interrupted by calling InterruptSocks5(bool). - * Sockets can be made non-blocking with SetSocketNonBlocking(const - * SOCKET&). + * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { @@ -503,7 +502,7 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) // Ensure that waiting for I/O on this socket won't result in undefined // behavior. - if (!IsSelectableSocket(sock->Get())) { + if (!sock->IsSelectable()) { LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); return nullptr; } @@ -525,7 +524,7 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) } // Set the non-blocking option on the socket. - if (!SetSocketNonBlocking(sock->Get())) { + if (!sock->SetNonBlocking()) { LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); return nullptr; } @@ -717,21 +716,6 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) return false; } -bool SetSocketNonBlocking(const SOCKET& hSocket) -{ -#ifdef WIN32 - u_long nOne = 1; - if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { -#else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { -#endif - return false; - } - - return true; -} - void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index fadc8b418e..f7816f5d1d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -221,8 +221,6 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -/** Enable non-blocking mode for a socket */ -bool SetSocketNonBlocking(const SOCKET& hSocket); void InterruptSocks5(bool interrupt); /** diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 57f81e6bb6..04d46f4361 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -524,6 +524,16 @@ void BlockManager::FlushUndoFile(int block_file, bool finalize) void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) { LOCK(cs_LastBlockFile); + + if (m_blockfile_info.size() < 1) { + // Return if we haven't loaded any blockfiles yet. This happens during + // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which + // then calls FlushStateToDisk()), resulting in a call to this function before we + // have populated `m_blockfile_info` via LoadBlockIndexDB(). + return; + } + assert(static_cast<int>(m_blockfile_info.size()) > m_last_blockfile); + FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); @@ -789,19 +799,24 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c return true; } -/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; - if (dbp != nullptr) { + const auto position_known {dbp != nullptr}; + if (position_known) { blockPos = *dbp; + } else { + // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for + // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). + // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. + nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } - if (dbp == nullptr) { + if (!position_known) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 37d74ed102..29501c1959 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -44,6 +44,9 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB +/** Size of header written by WriteBlockToDisk before a serialized CBlock */ +static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); + extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; /** Pruning-related variables and constants */ @@ -171,6 +174,7 @@ public: bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); /** Calculate the amount of disk space the block & undo files currently use */ diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 3f1d6dd743..26af523491 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -48,10 +48,15 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } LOCK(cs_main); - chainman.InitializeChainstate(options.mempool); chainman.m_total_coinstip_cache = cache_sizes.coins; chainman.m_total_coinsdb_cache = cache_sizes.coins_db; + // Load the fully validated chainstate. + chainman.InitializeChainstate(options.mempool); + + // Load a chain created from a UTXO snapshot, if any exist. + chainman.DetectSnapshotChainstate(options.mempool); + auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -98,12 +103,20 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } + // Conservative value which is arbitrarily chosen, as it will ultimately be changed + // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure + // that the sum of the two caches (40%) does not exceed the allowable amount + // during this temporary initialization state. + double init_cache_fraction = 0.2; + // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! for (Chainstate* chainstate : chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( - /*cache_size_bytes=*/cache_sizes.coins_db, + /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, /*should_wipe=*/options.reindex || options.reindex_chainstate); @@ -125,7 +138,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(cache_sizes.coins); + chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { @@ -146,6 +159,11 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize }; } + // Now that chainstates are loaded and we're able to flush to + // disk, rebalance the coins caches to desired levels based + // on the condition of each chainstate. + chainman.MaybeRebalanceCaches(); + return {ChainstateLoadStatus::SUCCESS, {}}; } diff --git a/src/node/txreconciliation.cpp b/src/node/txreconciliation.cpp new file mode 100644 index 0000000000..974358fcda --- /dev/null +++ b/src/node/txreconciliation.cpp @@ -0,0 +1,184 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txreconciliation.h> + +#include <util/check.h> +#include <util/system.h> + +#include <unordered_map> +#include <variant> + + +namespace { + +/** Static salt component used to compute short txids for sketch construction, see BIP-330. */ +const std::string RECON_STATIC_SALT = "Tx Relay Salting"; +const HashWriter RECON_SALT_HASHER = TaggedHash(RECON_STATIC_SALT); + +/** + * Salt (specified by BIP-330) constructed from contributions from both peers. It is used + * to compute transaction short IDs, which are then used to construct a sketch representing a set + * of transactions we want to announce to the peer. + */ +uint256 ComputeSalt(uint64_t salt1, uint64_t salt2) +{ + // According to BIP-330, salts should be combined in ascending order. + return (HashWriter(RECON_SALT_HASHER) << std::min(salt1, salt2) << std::max(salt1, salt2)).GetSHA256(); +} + +/** + * Keeps track of txreconciliation-related per-peer state. + */ +class TxReconciliationState +{ +public: + /** + * TODO: This field is public to ignore -Wunused-private-field. Make private once used in + * the following commits. + * + * Reconciliation protocol assumes using one role consistently: either a reconciliation + * initiator (requesting sketches), or responder (sending sketches). This defines our role. + * + */ + bool m_we_initiate; + + /** + * TODO: These fields are public to ignore -Wunused-private-field. Make private once used in + * the following commits. + * + * These values are used to salt short IDs, which is necessary for transaction reconciliations. + */ + uint64_t m_k0, m_k1; + + TxReconciliationState(bool we_initiate, uint64_t k0, uint64_t k1) : m_we_initiate(we_initiate), m_k0(k0), m_k1(k1) {} +}; + +} // namespace + +/** Actual implementation for TxReconciliationTracker's data structure. */ +class TxReconciliationTracker::Impl +{ +private: + mutable Mutex m_txreconciliation_mutex; + + // Local protocol version + uint32_t m_recon_version; + + /** + * Keeps track of txreconciliation states of eligible peers. + * For pre-registered peers, the locally generated salt is stored. + * For registered peers, the locally generated salt is forgotten, and the state (including + * "full" salt) is stored instead. + */ + std::unordered_map<NodeId, std::variant<uint64_t, TxReconciliationState>> m_states GUARDED_BY(m_txreconciliation_mutex); + +public: + explicit Impl(uint32_t recon_version) : m_recon_version(recon_version) {} + + uint64_t PreRegisterPeer(NodeId peer_id) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + // We do not support txreconciliation salt/version updates. + assert(m_states.find(peer_id) == m_states.end()); + + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Pre-register peer=%d\n", peer_id); + const uint64_t local_salt{GetRand(UINT64_MAX)}; + + // We do this exactly once per peer (which are unique by NodeId, see GetNewNodeId) so it's + // safe to assume we don't have this record yet. + Assert(m_states.emplace(peer_id, local_salt).second); + return local_salt; + } + + ReconciliationRegisterResult RegisterPeer(NodeId peer_id, bool is_peer_inbound, bool is_peer_recon_initiator, + bool is_peer_recon_responder, uint32_t peer_recon_version, + uint64_t remote_salt) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + auto recon_state = m_states.find(peer_id); + + // A peer should be in the pre-registered state to proceed here. + if (recon_state == m_states.end()) return NOT_FOUND; + uint64_t* local_salt = std::get_if<uint64_t>(&recon_state->second); + // A peer is already registered. This should be checked by the caller. + Assume(local_salt); + + // If the peer supports the version which is lower than ours, we downgrade to the version + // it supports. For now, this only guarantees that nodes with future reconciliation + // versions have the choice of reconciling with this current version. However, they also + // have the choice to refuse supporting reconciliations if the common version is not + // satisfactory (e.g. too low). + const uint32_t recon_version{std::min(peer_recon_version, m_recon_version)}; + // v1 is the lowest version, so suggesting something below must be a protocol violation. + if (recon_version < 1) return PROTOCOL_VIOLATION; + + // Must match SENDTXRCNCL logic. + const bool they_initiate = is_peer_recon_initiator && is_peer_inbound; + const bool we_initiate = !is_peer_inbound && is_peer_recon_responder; + + // If we ever announce support for both requesting and responding, this will need + // tie-breaking. For now, this is mutually exclusive because both are based on the + // inbound flag. + assert(!(they_initiate && we_initiate)); + + // The peer set both flags to false, we treat it as a protocol violation. + if (!(they_initiate || we_initiate)) return PROTOCOL_VIOLATION; + + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Register peer=%d with the following params: " /* Continued */ + "we_initiate=%i, they_initiate=%i.\n", + peer_id, we_initiate, they_initiate); + + const uint256 full_salt{ComputeSalt(*local_salt, remote_salt)}; + recon_state->second = TxReconciliationState(we_initiate, full_salt.GetUint64(0), full_salt.GetUint64(1)); + return SUCCESS; + } + + void ForgetPeer(NodeId peer_id) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + if (m_states.erase(peer_id)) { + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Forget txreconciliation state of peer=%d\n", peer_id); + } + } + + bool IsPeerRegistered(NodeId peer_id) const EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + auto recon_state = m_states.find(peer_id); + return (recon_state != m_states.end() && + std::holds_alternative<TxReconciliationState>(recon_state->second)); + } +}; + +TxReconciliationTracker::TxReconciliationTracker(uint32_t recon_version) : m_impl{std::make_unique<TxReconciliationTracker::Impl>(recon_version)} {} + +TxReconciliationTracker::~TxReconciliationTracker() = default; + +uint64_t TxReconciliationTracker::PreRegisterPeer(NodeId peer_id) +{ + return m_impl->PreRegisterPeer(peer_id); +} + +ReconciliationRegisterResult TxReconciliationTracker::RegisterPeer(NodeId peer_id, bool is_peer_inbound, + bool is_peer_recon_initiator, bool is_peer_recon_responder, + uint32_t peer_recon_version, uint64_t remote_salt) +{ + return m_impl->RegisterPeer(peer_id, is_peer_inbound, is_peer_recon_initiator, is_peer_recon_responder, + peer_recon_version, remote_salt); +} + +void TxReconciliationTracker::ForgetPeer(NodeId peer_id) +{ + m_impl->ForgetPeer(peer_id); +} + +bool TxReconciliationTracker::IsPeerRegistered(NodeId peer_id) const +{ + return m_impl->IsPeerRegistered(peer_id); +} diff --git a/src/node/txreconciliation.h b/src/node/txreconciliation.h new file mode 100644 index 0000000000..a4f0870914 --- /dev/null +++ b/src/node/txreconciliation.h @@ -0,0 +1,90 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TXRECONCILIATION_H +#define BITCOIN_NODE_TXRECONCILIATION_H + +#include <net.h> +#include <sync.h> + +#include <memory> +#include <tuple> + +/** Whether transaction reconciliation protocol should be enabled by default. */ +static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; +/** Supported transaction reconciliation protocol version */ +static constexpr uint32_t TXRECONCILIATION_VERSION{1}; + +enum ReconciliationRegisterResult { + NOT_FOUND = 0, + SUCCESS = 1, + PROTOCOL_VIOLATION = 2, +}; + +/** + * Transaction reconciliation is a way for nodes to efficiently announce transactions. + * This object keeps track of all txreconciliation-related communications with the peers. + * The high-level protocol is: + * 0. Txreconciliation protocol handshake. + * 1. Once we receive a new transaction, add it to the set instead of announcing immediately. + * 2. At regular intervals, a txreconciliation initiator requests a sketch from a peer, where a + * sketch is a compressed representation of short form IDs of the transactions in their set. + * 3. Once the initiator received a sketch from the peer, the initiator computes a local sketch, + * and combines the two sketches to attempt finding the difference in *sets*. + * 4a. If the difference was not larger than estimated, see SUCCESS below. + * 4b. If the difference was larger than estimated, initial txreconciliation fails. The initiator + * requests a larger sketch via an extension round (allowed only once). + * - If extension succeeds (a larger sketch is sufficient), see SUCCESS below. + * - If extension fails (a larger sketch is insufficient), see FAILURE below. + * + * SUCCESS. The initiator knows full symmetrical difference and can request what the initiator is + * missing and announce to the peer what the peer is missing. + * + * FAILURE. The initiator notifies the peer about the failure and announces all transactions from + * the corresponding set. Once the peer received the failure notification, the peer + * announces all transactions from their set. + + * This is a modification of the Erlay protocol (https://arxiv.org/abs/1905.10518) with two + * changes (sketch extensions instead of bisections, and an extra INV exchange round), both + * are motivated in BIP-330. + */ +class TxReconciliationTracker +{ +private: + class Impl; + const std::unique_ptr<Impl> m_impl; + +public: + explicit TxReconciliationTracker(uint32_t recon_version); + ~TxReconciliationTracker(); + + /** + * Step 0. Generates initial part of the state (salt) required to reconcile txs with the peer. + * The salt is used for short ID computation required for txreconciliation. + * The function returns the salt. + * A peer can't participate in future txreconciliations without this call. + * This function must be called only once per peer. + */ + uint64_t PreRegisterPeer(NodeId peer_id); + + /** + * Step 0. Once the peer agreed to reconcile txs with us, generate the state required to track + * ongoing reconciliations. Must be called only after pre-registering the peer and only once. + */ + ReconciliationRegisterResult RegisterPeer(NodeId peer_id, bool is_peer_inbound, bool is_peer_recon_initiator, + bool is_peer_recon_responder, uint32_t peer_recon_version, uint64_t remote_salt); + + /** + * Attempts to forget txreconciliation-related state of the peer (if we previously stored any). + * After this, we won't be able to reconcile transactions with the peer. + */ + void ForgetPeer(NodeId peer_id); + + /** + * Check if a peer is registered to reconcile transactions with us. + */ + bool IsPeerRegistered(NodeId peer_id) const; +}; + +#endif // BITCOIN_NODE_TXRECONCILIATION_H diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp new file mode 100644 index 0000000000..bab1b75211 --- /dev/null +++ b/src/node/utxo_snapshot.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/utxo_snapshot.h> + +#include <fs.h> +#include <logging.h> +#include <streams.h> +#include <uint256.h> +#include <util/system.h> +#include <validation.h> + +#include <cstdio> +#include <optional> + +namespace node { + +bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) +{ + AssertLockHeld(::cs_main); + assert(snapshot_chainstate.m_from_snapshot_blockhash); + + const std::optional<fs::path> chaindir = snapshot_chainstate.CoinsDB().StoragePath(); + assert(chaindir); // Sanity check that chainstate isn't in-memory. + const fs::path write_to = *chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; + + FILE* file{fsbridge::fopen(write_to, "wb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + LogPrintf("[snapshot] failed to open base blockhash file for writing: %s\n", + fs::PathToString(write_to)); + return false; + } + afile << *snapshot_chainstate.m_from_snapshot_blockhash; + + if (afile.fclose() != 0) { + LogPrintf("[snapshot] failed to close base blockhash file %s after writing\n", + fs::PathToString(write_to)); + return false; + } + return true; +} + +std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) +{ + if (!fs::exists(chaindir)) { + LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " /* Continued */ + "exists at path %s\n", fs::PathToString(chaindir)); + return std::nullopt; + } + const fs::path read_from = chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; + const std::string read_from_str = fs::PathToString(read_from); + + if (!fs::exists(read_from)) { + LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " /* Continued */ + "exists at path %s. Try deleting %s and calling loadtxoutset again?\n", + fs::PathToString(chaindir), read_from_str); + return std::nullopt; + } + + uint256 base_blockhash; + FILE* file{fsbridge::fopen(read_from, "rb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + LogPrintf("[snapshot] failed to open base blockhash file for reading: %s\n", + read_from_str); + return std::nullopt; + } + afile >> base_blockhash; + + if (std::fgetc(afile.Get()) != EOF) { + LogPrintf("[snapshot] warning: unexpected trailing data in %s\n", read_from_str); + } else if (std::ferror(afile.Get())) { + LogPrintf("[snapshot] warning: i/o error reading %s\n", read_from_str); + } + return base_blockhash; +} + +std::optional<fs::path> FindSnapshotChainstateDir() +{ + fs::path possible_dir = + gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); + + if (fs::exists(possible_dir)) { + return possible_dir; + } + return std::nullopt; +} + +} // namespace node diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 9dd6f06997..c94521792f 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -6,8 +6,14 @@ #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include <fs.h> #include <uint256.h> #include <serialize.h> +#include <validation.h> + +#include <optional> + +extern RecursiveMutex cs_main; namespace node { //! Metadata describing a serialized version of a UTXO set from which an @@ -33,6 +39,33 @@ public: SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } }; + +//! The file in the snapshot chainstate dir which stores the base blockhash. This is +//! needed to reconstruct snapshot chainstates on init. +//! +//! Because we only allow loading a single snapshot at a time, there will only be one +//! chainstate directory with this filename present within it. +const fs::path SNAPSHOT_BLOCKHASH_FILENAME{"base_blockhash"}; + +//! Write out the blockhash of the snapshot base block that was used to construct +//! this chainstate. This value is read in during subsequent initializations and +//! used to reconstruct snapshot-based chainstates. +bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +//! Read the blockhash of the snapshot base block that was used to construct the +//! chainstate. +std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +//! Suffix appended to the chainstate (leveldb) dir when created based upon +//! a snapshot. +constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; + + +//! Return a path to the snapshot-based chainstate dir, if one exists. +std::optional<fs::path> FindSnapshotChainstateDir(); + } // namespace node #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 22defb91a9..ab5599a1b4 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1011,20 +1011,37 @@ void CBlockPolicyEstimator::FlushUnconfirmed() LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear)); } -FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) +static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee, + double max_filter_fee_rate, + double fee_filter_spacing) { - CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); - feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { - feeset.insert(bucketBoundary); + std::set<double> fee_set; + + const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)}; + fee_set.insert(0); + for (double bucket_boundary = min_fee_limit; + bucket_boundary <= max_filter_fee_rate; + bucket_boundary *= fee_filter_spacing) { + + fee_set.insert(bucket_boundary); } + + return fee_set; +} + +FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) + : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} +{ } CAmount FeeFilterRounder::round(CAmount currentMinFee) { - std::set<double>::iterator it = feeset.lower_bound(currentMinFee); - if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { - it--; + AssertLockNotHeld(m_insecure_rand_mutex); + std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee); + if (it == m_fee_set.end() || + (it != m_fee_set.begin() && + WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { + --it; } return static_cast<CAmount>(*it); } diff --git a/src/policy/fees.h b/src/policy/fees.h index e4628bf853..204c4f2118 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -299,14 +299,15 @@ private: public: /** Create new FeeFilterRounder */ - explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); + explicit FeeFilterRounder(const CFeeRate& min_incremental_fee); - /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ - CAmount round(CAmount currentMinFee); + /** Quantize a minimum fee for privacy purpose before broadcast. */ + CAmount round(CAmount currentMinFee) EXCLUSIVE_LOCKS_REQUIRED(!m_insecure_rand_mutex); private: - std::set<double> feeset; - FastRandomContext insecure_rand; + const std::set<double> m_fee_set; + Mutex m_insecure_rand_mutex; + FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; #endif // BITCOIN_POLICY_FEES_H diff --git a/src/protocol.cpp b/src/protocol.cpp index bdd1cc2aff..23c68b335b 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -44,6 +44,7 @@ const char *CFHEADERS="cfheaders"; const char *GETCFCHECKPT="getcfcheckpt"; const char *CFCHECKPT="cfcheckpt"; const char *WTXIDRELAY="wtxidrelay"; +const char *SENDTXRCNCL="sendtxrcncl"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -84,6 +85,7 @@ const static std::string allNetMessageTypes[] = { NetMsgType::GETCFCHECKPT, NetMsgType::CFCHECKPT, NetMsgType::WTXIDRELAY, + NetMsgType::SENDTXRCNCL, }; const static std::vector<std::string> allNetMessageTypesVec(std::begin(allNetMessageTypes), std::end(allNetMessageTypes)); diff --git a/src/protocol.h b/src/protocol.h index b85dc0d820..17a363b1d3 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -258,6 +258,14 @@ extern const char* CFCHECKPT; * @since protocol version 70016 as described by BIP 339. */ extern const char* WTXIDRELAY; +/** + * Contains 2 1-byte bools, a 4-byte version number and an 8-byte salt. + * The 2 booleans indicate that a node is willing to participate in transaction + * reconciliation, respectively as an initiator or as a receiver. + * The salt is used to compute short txids needed for efficient + * txreconciliation, as described by BIP 330. + */ +extern const char* SENDTXRCNCL; }; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ diff --git a/src/psbt.cpp b/src/psbt.cpp index 36fec74bc9..cbf2f88788 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -218,8 +218,14 @@ void PSBTOutput::FillSignatureData(SignatureData& sigdata) const for (const auto& key_pair : hd_keypaths) { sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); } - if (m_tap_tree.has_value() && m_tap_internal_key.IsFullyValid()) { - TaprootSpendData spenddata = m_tap_tree->GetSpendData(); + if (!m_tap_tree.empty() && m_tap_internal_key.IsFullyValid()) { + TaprootBuilder builder; + for (const auto& [depth, leaf_ver, script] : m_tap_tree) { + builder.Add((int)depth, script, (int)leaf_ver, /*track=*/true); + } + assert(builder.IsComplete()); + builder.Finalize(m_tap_internal_key); + TaprootSpendData spenddata = builder.GetSpendData(); sigdata.tr_spenddata.internal_key = m_tap_internal_key; sigdata.tr_spenddata.Merge(spenddata); @@ -243,8 +249,8 @@ void PSBTOutput::FromSignatureData(const SignatureData& sigdata) if (!sigdata.tr_spenddata.internal_key.IsNull()) { m_tap_internal_key = sigdata.tr_spenddata.internal_key; } - if (sigdata.tr_builder.has_value()) { - m_tap_tree = sigdata.tr_builder; + if (sigdata.tr_builder.has_value() && sigdata.tr_builder->HasScripts()) { + m_tap_tree = sigdata.tr_builder->GetTreeTuples(); } for (const auto& [pubkey, leaf_origin] : sigdata.taproot_misc_pubkeys) { m_tap_bip32_paths.emplace(pubkey, leaf_origin); @@ -265,7 +271,7 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; if (m_tap_internal_key.IsNull() && !output.m_tap_internal_key.IsNull()) m_tap_internal_key = output.m_tap_internal_key; - if (m_tap_tree.has_value() && !output.m_tap_tree.has_value()) m_tap_tree = output.m_tap_tree; + if (m_tap_tree.empty() && !output.m_tap_tree.empty()) m_tap_tree = output.m_tap_tree; } bool PSBTInputSigned(const PSBTInput& input) { diff --git a/src/psbt.h b/src/psbt.h index d5c67802c7..ddcdb8c68d 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -713,7 +713,7 @@ struct PSBTOutput CScript witness_script; std::map<CPubKey, KeyOriginInfo> hd_keypaths; XOnlyPubKey m_tap_internal_key; - std::optional<TaprootBuilder> m_tap_tree; + std::vector<std::tuple<uint8_t, uint8_t, CScript>> m_tap_tree; std::map<XOnlyPubKey, std::pair<std::set<uint256>, KeyOriginInfo>> m_tap_bip32_paths; std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; std::set<PSBTProprietary> m_proprietary; @@ -754,15 +754,11 @@ struct PSBTOutput } // Write taproot tree - if (m_tap_tree.has_value()) { + if (!m_tap_tree.empty()) { SerializeToVector(s, PSBT_OUT_TAP_TREE); std::vector<unsigned char> value; CVectorWriter s_value(s.GetType(), s.GetVersion(), value, 0); - const auto& tuples = m_tap_tree->GetTreeTuples(); - for (const auto& tuple : tuples) { - uint8_t depth = std::get<0>(tuple); - uint8_t leaf_ver = std::get<1>(tuple); - CScript script = std::get<2>(tuple); + for (const auto& [depth, leaf_ver, script] : m_tap_tree) { s_value << depth; s_value << leaf_ver; s_value << script; @@ -858,10 +854,13 @@ struct PSBTOutput } else if (key.size() != 1) { throw std::ios_base::failure("Output Taproot tree key is more than one byte type"); } - m_tap_tree.emplace(); std::vector<unsigned char> tree_v; s >> tree_v; SpanReader s_tree(s.GetType(), s.GetVersion(), tree_v); + if (s_tree.empty()) { + throw std::ios_base::failure("Output Taproot tree must not be empty"); + } + TaprootBuilder builder; while (!s_tree.empty()) { uint8_t depth; uint8_t leaf_ver; @@ -875,9 +874,10 @@ struct PSBTOutput if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); } - m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); + m_tap_tree.push_back(std::make_tuple(depth, leaf_ver, script)); + builder.Add((int)depth, script, (int)leaf_ver, true /* track */); } - if (!m_tap_tree->IsComplete()) { + if (!builder.IsComplete()) { throw std::ios_base::failure("Output Taproot tree is malformed"); } break; @@ -931,11 +931,6 @@ struct PSBTOutput } } - // Finalize m_tap_tree so that all of the computed things are computed - if (m_tap_tree.has_value() && m_tap_tree->IsComplete() && m_tap_internal_key.IsFullyValid()) { - m_tap_tree->Finalize(m_tap_internal_key); - } - if (!found_sep) { throw std::ios_base::failure("Separator is missing at the end of an output map"); } diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 582f02132a..6be47e94fe 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -286,7 +286,7 @@ <item> <widget class="QLineEdit" name="externalSignerPath"> <property name="toolTip"> - <string>Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> + <string>Full path to a %1 compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> </property> </widget> </item> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 2b6711ca40..7c91eac37e 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -91,7 +91,9 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->thirdPartyTxUrls->setVisible(false); } -#ifndef ENABLE_EXTERNAL_SIGNER +#ifdef ENABLE_EXTERNAL_SIGNER + ui->externalSignerPath->setToolTip(ui->externalSignerPath->toolTip().arg(PACKAGE_NAME)); +#else //: "External signing" means using devices such as hardware wallets. ui->externalSignerPath->setToolTip(tr("Compiled without external signing support (required for external signing)")); ui->externalSignerPath->setEnabled(false); diff --git a/src/rest.cpp b/src/rest.cpp index 7f00db2222..a10d8a433f 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -590,6 +590,48 @@ static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std: } } + +RPCHelpMan getdeploymentinfo(); + +static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) +{ + if (!CheckWarmup(req)) return false; + + std::string hash_str; + const RESTResponseFormat rf = ParseDataFormat(hash_str, str_uri_part); + + switch (rf) { + case RESTResponseFormat::JSON: { + JSONRPCRequest jsonRequest; + jsonRequest.context = context; + jsonRequest.params = UniValue(UniValue::VARR); + + if (!hash_str.empty()) { + uint256 hash; + if (!ParseHashStr(hash_str, hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hash_str); + } + + const ChainstateManager* chainman = GetChainman(context, req); + if (!chainman) return false; + if (!WITH_LOCK(::cs_main, return chainman->m_blockman.LookupBlockIndex(ParseHashV(hash_str, "blockhash")))) { + return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); + } + + jsonRequest.params.pushKV("blockhash", hash_str); + } + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, getdeploymentinfo().HandleRequest(jsonRequest).write() + "\n"); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + +} + static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) @@ -939,6 +981,8 @@ static const struct { {"/rest/mempool/", rest_mempool}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, + {"/rest/deploymentinfo/", rest_deploymentinfo}, + {"/rest/deploymentinfo", rest_deploymentinfo}, {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 93d11c8276..f8ba822f54 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1307,7 +1307,7 @@ UniValue DeploymentInfo(const CBlockIndex* blockindex, const ChainstateManager& } } // anon namespace -static RPCHelpMan getdeploymentinfo() +RPCHelpMan getdeploymentinfo() { return RPCHelpMan{"getdeploymentinfo", "Returns an object containing various state info regarding deployments of consensus changes.", @@ -2019,6 +2019,40 @@ public: } }; +static const auto scan_action_arg_desc = RPCArg{ + "action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" + "\"start\" for starting a scan\n" + "\"abort\" for aborting the current scan (returns true when abort was successful)\n" + "\"status\" for progress report (in %) of the current scan" +}; + +static const auto scan_objects_arg_desc = RPCArg{ + "scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" + "Every scan object is either a string descriptor or an object:", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"}, + }}, + }, + RPCArgOptions{.oneline_description="[scanobjects,...]"}, +}; + +static const auto scan_result_abort = RPCResult{ + "when action=='abort'", RPCResult::Type::BOOL, "success", + "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort" +}; +static const auto scan_result_status_none = RPCResult{ + "when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", "" +}; +static const auto scan_result_status_some = RPCResult{ + "when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", + {{RPCResult::Type::NUM, "progress", "Approximate percent complete"},} +}; + + static RPCHelpMan scantxoutset() { // scriptPubKey corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S @@ -2038,21 +2072,8 @@ static RPCHelpMan scantxoutset() "In the latter case, a range needs to be specified by below if different from 1000.\n" "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n", { - {"action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" - "\"start\" for starting a scan\n" - "\"abort\" for aborting the current scan (returns true when abort was successful)\n" - "\"status\" for progress report (in %) of the current scan"}, - {"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" - "Every scan object is either a string descriptor or an object:", - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", - { - {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, - {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"}, - }}, - }, - RPCArgOptions{.oneline_description="[scanobjects,...]"}}, + scan_action_arg_desc, + scan_objects_arg_desc, }, { RPCResult{"when action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", { @@ -2069,17 +2090,15 @@ static RPCHelpMan scantxoutset() {RPCResult::Type::STR_HEX, "scriptPubKey", "The script key"}, {RPCResult::Type::STR, "desc", "A specialized descriptor for the matched scriptPubKey"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the unspent output"}, + {RPCResult::Type::BOOL, "coinbase", "Whether this is a coinbase output"}, {RPCResult::Type::NUM, "height", "Height of the unspent transaction output"}, }}, }}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT}, }}, - RPCResult{"when action=='abort'", RPCResult::Type::BOOL, "success", "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort"}, - RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, - }}, - RPCResult{"when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", ""}, + scan_result_abort, + scan_result_status_some, + scan_result_status_none, }, RPCExamples{ HelpExampleCli("scantxoutset", "start \'[\"" + EXAMPLE_DESCRIPTOR_RAW + "\"]\'") + @@ -2172,6 +2191,7 @@ static RPCHelpMan scantxoutset() unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); + unspent.pushKV("coinbase", coin.IsCoinBase()); unspent.pushKV("height", (int32_t)coin.nHeight); unspents.push_back(unspent); @@ -2186,6 +2206,203 @@ static RPCHelpMan scantxoutset() }; } +/** RAII object to prevent concurrency issue when scanning blockfilters */ +static std::atomic<int> g_scanfilter_progress; +static std::atomic<int> g_scanfilter_progress_height; +static std::atomic<bool> g_scanfilter_in_progress; +static std::atomic<bool> g_scanfilter_should_abort_scan; +class BlockFiltersScanReserver +{ +private: + bool m_could_reserve{false}; +public: + explicit BlockFiltersScanReserver() = default; + + bool reserve() { + CHECK_NONFATAL(!m_could_reserve); + if (g_scanfilter_in_progress.exchange(true)) { + return false; + } + m_could_reserve = true; + return true; + } + + ~BlockFiltersScanReserver() { + if (m_could_reserve) { + g_scanfilter_in_progress = false; + } + } +}; + +static RPCHelpMan scanblocks() +{ + return RPCHelpMan{"scanblocks", + "\nReturn relevant blockhashes for given descriptors.\n" + "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", + { + scan_action_arg_desc, + scan_objects_arg_desc, + RPCArg{"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "Height to start to scan from"}, + RPCArg{"stop_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"chain tip"}, "Height to stop to scan"}, + RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC)}, "The type name of the filter"} + }, + { + scan_result_status_none, + RPCResult{"When action=='start'", RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "from_height", "The height we started the scan from"}, + {RPCResult::Type::NUM, "to_height", "The height we ended the scan at"}, + {RPCResult::Type::ARR, "relevant_blocks", "", {{RPCResult::Type::STR_HEX, "blockhash", "A relevant blockhash"},}}, + }, + }, + RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, + {RPCResult::Type::NUM, "current_height", "Height of the block currently being scanned"}, + }, + }, + scan_result_abort, + }, + RPCExamples{ + HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 300000") + + HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 100 150 basic") + + HelpExampleCli("scanblocks", "status") + + HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 300000") + + HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 100, 150, \"basic\"") + + HelpExampleRpc("scanblocks", "\"status\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue ret(UniValue::VOBJ); + if (request.params[0].get_str() == "status") { + BlockFiltersScanReserver reserver; + if (reserver.reserve()) { + // no scan in progress + return NullUniValue; + } + ret.pushKV("progress", g_scanfilter_progress.load()); + ret.pushKV("current_height", g_scanfilter_progress_height.load()); + return ret; + } else if (request.params[0].get_str() == "abort") { + BlockFiltersScanReserver reserver; + if (reserver.reserve()) { + // reserve was possible which means no scan was running + return false; + } + // set the abort flag + g_scanfilter_should_abort_scan = true; + return true; + } + else if (request.params[0].get_str() == "start") { + BlockFiltersScanReserver reserver; + if (!reserver.reserve()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); + } + const std::string filtertype_name{request.params[4].isNull() ? "basic" : request.params[4].get_str()}; + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(filtertype_name, filtertype)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype"); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = EnsureChainman(node); + + // set the start-height + const CBlockIndex* block = nullptr; + const CBlockIndex* stop_block = nullptr; + { + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + block = active_chain.Genesis(); + stop_block = active_chain.Tip(); + if (!request.params[2].isNull()) { + block = active_chain[request.params[2].getInt<int>()]; + if (!block) { + throw JSONRPCError(RPC_MISC_ERROR, "Invalid start_height"); + } + } + if (!request.params[3].isNull()) { + stop_block = active_chain[request.params[3].getInt<int>()]; + if (!stop_block || stop_block->nHeight < block->nHeight) { + throw JSONRPCError(RPC_MISC_ERROR, "Invalid stop_height"); + } + } + } + CHECK_NONFATAL(block); + + // loop through the scan objects, add scripts to the needle_set + GCSFilter::ElementSet needle_set; + for (const UniValue& scanobject : request.params[1].get_array().getValues()) { + FlatSigningProvider provider; + std::vector<CScript> scripts = EvalDescriptorStringOrObject(scanobject, provider); + for (const CScript& script : scripts) { + needle_set.emplace(script.begin(), script.end()); + } + } + UniValue blocks(UniValue::VARR); + const int amount_per_chunk = 10000; + const CBlockIndex* start_index = block; // for remembering the start of a blockfilter range + std::vector<BlockFilter> filters; + const CBlockIndex* start_block = block; // for progress reporting + const int total_blocks_to_process = stop_block->nHeight - start_block->nHeight; + + g_scanfilter_should_abort_scan = false; + g_scanfilter_progress = 0; + g_scanfilter_progress_height = start_block->nHeight; + + while (block) { + node.rpc_interruption_point(); // allow a clean shutdown + if (g_scanfilter_should_abort_scan) { + LogPrintf("scanblocks RPC aborted at height %d.\n", block->nHeight); + break; + } + const CBlockIndex* next = nullptr; + { + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + next = active_chain.Next(block); + if (block == stop_block) next = nullptr; + } + if (start_index->nHeight + amount_per_chunk == block->nHeight || next == nullptr) { + LogPrint(BCLog::RPC, "Fetching blockfilters from height %d to height %d.\n", start_index->nHeight, block->nHeight); + if (index->LookupFilterRange(start_index->nHeight, block, filters)) { + for (const BlockFilter& filter : filters) { + // compare the elements-set with each filter + if (filter.GetFilter().MatchAny(needle_set)) { + blocks.push_back(filter.GetBlockHash().GetHex()); + LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); + } + } + } + start_index = block; + + // update progress + int blocks_processed = block->nHeight - start_block->nHeight; + if (total_blocks_to_process > 0) { // avoid division by zero + g_scanfilter_progress = (int)(100.0 / total_blocks_to_process * blocks_processed); + } else { + g_scanfilter_progress = 100; + } + g_scanfilter_progress_height = block->nHeight; + } + block = next; + } + ret.pushKV("from_height", start_block->nHeight); + ret.pushKV("to_height", g_scanfilter_progress_height.load()); + ret.pushKV("relevant_blocks", blocks); + } + else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid command"); + } + return ret; +}, + }; +} + static RPCHelpMan getblockfilter() { return RPCHelpMan{"getblockfilter", @@ -2421,6 +2638,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &verifychain}, {"blockchain", &preciousblock}, {"blockchain", &scantxoutset}, + {"blockchain", &scanblocks}, {"blockchain", &getblockfilter}, {"hidden", &invalidateblock}, {"hidden", &reconsiderblock}, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 612dbbdacf..8688263ef5 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -83,6 +83,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 8, "fee_rate"}, { "sendmany", 9, "verbose" }, { "deriveaddresses", 1, "range" }, + { "scanblocks", 1, "scanobjects" }, + { "scanblocks", 2, "start_height" }, + { "scanblocks", 3, "stop_height" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index d701a180ab..8d7f4e7f5b 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -114,7 +114,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether peer has asked us to relay transactions to it"}, + {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether we relay transactions to this peer"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, @@ -123,9 +123,9 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, - {RPCResult::Type::NUM, "pingtime", /*optional=*/true, "ping time (if available)"}, - {RPCResult::Type::NUM, "minping", /*optional=*/true, "minimum observed ping time (if any at all)"}, - {RPCResult::Type::NUM, "pingwait", /*optional=*/true, "ping wait (if non-zero)"}, + {RPCResult::Type::NUM, "pingtime", /*optional=*/true, "The last ping time in milliseconds (ms), if any"}, + {RPCResult::Type::NUM, "minping", /*optional=*/true, "The minimum observed ping time in milliseconds (ms), if any"}, + {RPCResult::Type::NUM, "pingwait", /*optional=*/true, "The duration in milliseconds (ms) of an outstanding ping (if non-zero)"}, {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, {RPCResult::Type::STR, "subver", "The string version"}, {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, @@ -146,7 +146,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, }}, - {RPCResult::Type::NUM, "minfeefilter", /*optional=*/true, "The minimum fee rate for transactions this peer accepts"}, + {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", { {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" @@ -200,6 +200,9 @@ static RPCHelpMan getpeerinfo() ServiceFlags services{fStateStats ? statestats.their_services : ServiceFlags::NODE_NONE}; obj.pushKV("services", strprintf("%016x", services)); obj.pushKV("servicesnames", GetServicesNames(services)); + if (fStateStats) { + obj.pushKV("relaytxes", statestats.m_relay_txs); + } obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -235,8 +238,6 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); - obj.pushKV("relaytxes", statestats.m_relay_txs); - obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); obj.pushKV("addr_processed", statestats.m_addr_processed); obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); @@ -246,6 +247,7 @@ static RPCHelpMan getpeerinfo() permissions.push_back(permission); } obj.pushKV("permissions", permissions); + obj.pushKV("minfeefilter", fStateStats ? ValueFromAmount(statestats.m_fee_filter_received) : 0); UniValue sendPerMsgType(UniValue::VOBJ); for (const auto& i : stats.mapSendBytesPerMsgType) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index f365de7d0c..d654de1862 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1241,13 +1241,9 @@ static RPCHelpMan decodepsbt() } // Taproot tree - if (output.m_tap_tree.has_value()) { + if (!output.m_tap_tree.empty()) { UniValue tree(UniValue::VARR); - const auto& tuples = output.m_tap_tree->GetTreeTuples(); - for (const auto& tuple : tuples) { - uint8_t depth = std::get<0>(tuple); - uint8_t leaf_ver = std::get<1>(tuple); - CScript script = std::get<2>(tuple); + for (const auto& [depth, leaf_ver, script] : output.m_tap_tree) { UniValue elem(UniValue::VOBJ); elem.pushKV("depth", (int)depth); elem.pushKV("leaf_ver", (int)leaf_ver); diff --git a/src/script/standard.h b/src/script/standard.h index 1e6769782a..966a52b2c7 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -315,6 +315,8 @@ public: TaprootSpendData GetSpendData() const; /** Returns a vector of tuples representing the depth, leaf version, and script */ std::vector<std::tuple<uint8_t, uint8_t, CScript>> GetTreeTuples() const; + /** Returns true if there are any tapscripts */ + bool HasScripts() const { return !m_branch.empty(); } }; /** Given a TaprootSpendData and the output key, reconstruct its script tree. diff --git a/src/streams.h b/src/streams.h index ce1ba26d45..0178df1c49 100644 --- a/src/streams.h +++ b/src/streams.h @@ -487,12 +487,14 @@ public: AutoFile(const AutoFile&) = delete; AutoFile& operator=(const AutoFile&) = delete; - void fclose() + int fclose() { + int retval{0}; if (file) { - ::fclose(file); + retval = ::fclose(file); file = nullptr; } + return retval; } /** Get wrapped FILE* with transfer of ownership. diff --git a/src/sync.h b/src/sync.h index a9d0091bd2..1f4e191214 100644 --- a/src/sync.h +++ b/src/sync.h @@ -111,7 +111,7 @@ public: return PARENT::try_lock(); } - using UniqueLock = std::unique_lock<PARENT>; + using unique_lock = std::unique_lock<PARENT>; #ifdef __clang__ //! For negative capabilities in the Clang Thread Safety Analysis. //! A negative requirement uses the EXCLUSIVE_LOCKS_REQUIRED attribute, in conjunction @@ -147,11 +147,13 @@ inline void AssertLockNotHeldInline(const char* name, const char* file, int line inline void AssertLockNotHeldInline(const char* name, const char* file, int line, GlobalMutex* cs) LOCKS_EXCLUDED(cs) { AssertLockNotHeldInternal(name, file, line, cs); } #define AssertLockNotHeld(cs) AssertLockNotHeldInline(#cs, __FILE__, __LINE__, &cs) -/** Wrapper around std::unique_lock style lock for Mutex. */ -template <typename Mutex, typename Base = typename Mutex::UniqueLock> -class SCOPED_LOCKABLE UniqueLock : public Base +/** Wrapper around std::unique_lock style lock for MutexType. */ +template <typename MutexType> +class SCOPED_LOCKABLE UniqueLock : public MutexType::unique_lock { private: + using Base = typename MutexType::unique_lock; + void Enter(const char* pszName, const char* pszFile, int nLine) { EnterCritical(pszName, pszFile, nLine, Base::mutex()); @@ -173,7 +175,7 @@ private: } public: - UniqueLock(Mutex& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : Base(mutexIn, std::defer_lock) + UniqueLock(MutexType& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : Base(mutexIn, std::defer_lock) { if (fTry) TryEnter(pszName, pszFile, nLine); @@ -181,7 +183,7 @@ public: Enter(pszName, pszFile, nLine); } - UniqueLock(Mutex* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) + UniqueLock(MutexType* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) { if (!pmutexIn) return; @@ -241,29 +243,24 @@ public: #define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock UNIQUE_NAME(revlock)(g, #g, __FILE__, __LINE__) -template<typename MutexArg> -using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; - // When locking a Mutex, require negative capability to ensure the lock // is not already held inline Mutex& MaybeCheckNotHeld(Mutex& cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) LOCK_RETURNED(cs) { return cs; } inline Mutex* MaybeCheckNotHeld(Mutex* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) LOCK_RETURNED(cs) { return cs; } -// When locking a GlobalMutex, just check it is not locked in the surrounding scope -inline GlobalMutex& MaybeCheckNotHeld(GlobalMutex& cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } -inline GlobalMutex* MaybeCheckNotHeld(GlobalMutex* cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } - -// When locking a RecursiveMutex, it's okay to already hold the lock -// but check that it is not known to be locked in the surrounding scope anyway -inline RecursiveMutex& MaybeCheckNotHeld(RecursiveMutex& cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } -inline RecursiveMutex* MaybeCheckNotHeld(RecursiveMutex* cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } +// When locking a GlobalMutex or RecursiveMutex, just check it is not +// locked in the surrounding scope. +template <typename MutexType> +inline MutexType& MaybeCheckNotHeld(MutexType& m) LOCKS_EXCLUDED(m) LOCK_RETURNED(m) { return m; } +template <typename MutexType> +inline MutexType* MaybeCheckNotHeld(MutexType* m) LOCKS_EXCLUDED(m) LOCK_RETURNED(m) { return m; } -#define LOCK(cs) DebugLock<decltype(cs)> UNIQUE_NAME(criticalblock)(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) +#define LOCK(cs) UniqueLock UNIQUE_NAME(criticalblock)(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) #define LOCK2(cs1, cs2) \ - DebugLock<decltype(cs1)> criticalblock1(MaybeCheckNotHeld(cs1), #cs1, __FILE__, __LINE__); \ - DebugLock<decltype(cs2)> criticalblock2(MaybeCheckNotHeld(cs2), #cs2, __FILE__, __LINE__) -#define TRY_LOCK(cs, name) DebugLock<decltype(cs)> name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__, true) -#define WAIT_LOCK(cs, name) DebugLock<decltype(cs)> name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) + UniqueLock criticalblock1(MaybeCheckNotHeld(cs1), #cs1, __FILE__, __LINE__); \ + UniqueLock criticalblock2(MaybeCheckNotHeld(cs2), #cs2, __FILE__, __LINE__) +#define TRY_LOCK(cs, name) UniqueLock name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__, true) +#define WAIT_LOCK(cs, name) UniqueLock name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) #define ENTER_CRITICAL_SECTION(cs) \ { \ diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 78b82b9b20..1c13c0a909 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/merkle.h> #include <pow.h> #include <streams.h> +#include <test/util/txmempool.h> #include <test/util/setup_common.h> diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp new file mode 100644 index 0000000000..dd7c32cc46 --- /dev/null +++ b/src/test/blockmanager_tests.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <node/blockstorage.h> +#include <node/context.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> + +using node::BlockManager; +using node::BLOCK_SERIALIZATION_HEADER_SIZE; + +// use BasicTestingSetup here for the data directory configuration, setup, and cleanup +BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) +{ + const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; + BlockManager blockman {}; + CChain chain {}; + // simulate adding a genesis block normally + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // simulate what happens during reindex + // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file + // the block is found at offset 8 because there is an 8 byte serialization header + // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. + FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // now simulate what happens after reindex for the first new block processed + // the actual block contents don't matter, just that it's a block. + // verify that the write position is at offset 0x12d. + // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur + // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 + // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 2a6a777cfe..8a2b0792fd 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -76,10 +76,16 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_CHECK(block_index != new_block_index); + // It is not safe to stop and destroy the index until it finishes handling + // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain() + // call above is sufficient to ensure this, but the + // SyncWithValidationInterfaceQueue() call below is also needed to ensure + // TSAN always sees the test thread waiting for the notification thread, and + // avoid potential false positive reports. + SyncWithValidationInterfaceQueue(); + // Shutdown sequence (c.f. Shutdown() in init.cpp) coin_stats_index.Stop(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() } // Test shutdown between BlockConnected and ChainStateFlushed notifications, diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index f6a35da4fc..5a4df735da 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -130,6 +130,7 @@ FUZZ_TARGET_MSG(pong); FUZZ_TARGET_MSG(sendaddrv2); FUZZ_TARGET_MSG(sendcmpct); FUZZ_TARGET_MSG(sendheaders); +FUZZ_TARGET_MSG(sendtxrcncl); FUZZ_TARGET_MSG(tx); FUZZ_TARGET_MSG(verack); FUZZ_TARGET_MSG(version); diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index e06e57d919..678fc7a5aa 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -11,6 +11,7 @@ #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <cstdint> diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 26913a41d2..f32046e69f 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -151,6 +151,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "preciousblock", "pruneblockchain", "reconsiderblock", + "scanblocks", "scantxoutset", "sendrawtransaction", "setmocktime", diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index a34e501fcc..46ca8e47e9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -13,6 +13,7 @@ #include <test/util/mining.h> #include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <util/rbf.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 0080b1e56e..d495a6bfe3 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -16,7 +16,7 @@ #include <memory> FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) - : m_fuzzed_data_provider{fuzzed_data_provider} + : m_fuzzed_data_provider{fuzzed_data_provider}, m_selectable{fuzzed_data_provider.ConsumeBool()} { m_socket = fuzzed_data_provider.ConsumeIntegralInRange<SOCKET>(INVALID_SOCKET - 1, INVALID_SOCKET); } @@ -254,6 +254,24 @@ int FuzzedSock::GetSockName(sockaddr* name, socklen_t* name_len) const return 0; } +bool FuzzedSock::SetNonBlocking() const +{ + constexpr std::array setnonblocking_errnos{ + EBADF, + EPERM, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, setnonblocking_errnos); + return false; + } + return true; +} + +bool FuzzedSock::IsSelectable() const +{ + return m_selectable; +} + bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { constexpr std::array wait_errnos{ diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index a354df2bf7..dfe4855326 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -47,6 +47,13 @@ class FuzzedSock : public Sock */ mutable std::optional<uint8_t> m_peek_data; + /** + * Whether to pretend that the socket is select(2)-able. This is randomly set in the + * constructor. It should remain constant so that repeated calls to `IsSelectable()` + * return the same value. + */ + const bool m_selectable; + public: explicit FuzzedSock(FuzzedDataProvider& fuzzed_data_provider); @@ -72,6 +79,10 @@ public: int GetSockName(sockaddr* name, socklen_t* name_len) const override; + bool SetNonBlocking() const override; + + bool IsSelectable() const override; + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; bool WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const override; diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index d96609416b..9a90de8911 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -12,6 +12,7 @@ #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/time.h> #include <validation.h> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 19f457b8dd..393311e4e4 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/policy.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/system.h> #include <util/time.h> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 696a799872..9bc29e3599 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -9,6 +9,7 @@ #include <node/miner.h> #include <policy/policy.h> #include <script/standard.h> +#include <test/util/txmempool.h> #include <timedata.h> #include <txmempool.h> #include <uint256.h> diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index c2d2fa37b4..0e1e9ae211 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -84,12 +84,12 @@ BOOST_AUTO_TEST_CASE(netbase_properties) } -bool static TestSplitHost(const std::string& test, const std::string& host, uint16_t port) +bool static TestSplitHost(const std::string& test, const std::string& host, uint16_t port, bool validPort=true) { std::string hostOut; uint16_t portOut{0}; - SplitHostPort(test, portOut, hostOut); - return hostOut == host && port == portOut; + bool validPortOut = SplitHostPort(test, portOut, hostOut); + return hostOut == host && portOut == port && validPortOut == validPort; } BOOST_AUTO_TEST_CASE(netbase_splithost) @@ -109,6 +109,23 @@ BOOST_AUTO_TEST_CASE(netbase_splithost) BOOST_CHECK(TestSplitHost(":8333", "", 8333)); BOOST_CHECK(TestSplitHost("[]:8333", "", 8333)); BOOST_CHECK(TestSplitHost("", "", 0)); + BOOST_CHECK(TestSplitHost(":65535", "", 65535)); + BOOST_CHECK(TestSplitHost(":65536", ":65536", 0, false)); + BOOST_CHECK(TestSplitHost(":-1", ":-1", 0, false)); + BOOST_CHECK(TestSplitHost("[]:70001", "[]:70001", 0, false)); + BOOST_CHECK(TestSplitHost("[]:-1", "[]:-1", 0, false)); + BOOST_CHECK(TestSplitHost("[]:-0", "[]:-0", 0, false)); + BOOST_CHECK(TestSplitHost("[]:0", "", 0, false)); + BOOST_CHECK(TestSplitHost("[]:1/2", "[]:1/2", 0, false)); + BOOST_CHECK(TestSplitHost("[]:1E2", "[]:1E2", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:65536", "127.0.0.1:65536", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:0", "127.0.0.1", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:", "127.0.0.1:", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:1/2", "127.0.0.1:1/2", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:1E2", "127.0.0.1:1E2", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:65536", "www.bitcoincore.org:65536", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:0", "www.bitcoincore.org", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:", "www.bitcoincore.org:", 0, false)); } bool static TestParse(std::string src, std::string canon) diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 3f66a8fc46..b652dc44c0 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -4,6 +4,7 @@ #include <policy/fees.h> #include <policy/policy.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <uint256.h> #include <util/time.h> diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index c88cd36688..d362c85560 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/rbf.h> #include <random.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/system.h> #include <util/time.h> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 55486db6b2..d78f62972f 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <coins.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <key.h> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 62c7ddb673..643d9221fe 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -69,11 +69,16 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) } } + // It is not safe to stop and destroy the index until it finishes handling + // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain() + // call above is sufficient to ensure this, but the + // SyncWithValidationInterfaceQueue() call below is also needed to ensure + // TSAN always sees the test thread waiting for the notification thread, and + // avoid potential false positive reports. + SyncWithValidationInterfaceQueue(); + // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - - // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed - SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txreconciliation_tests.cpp b/src/test/txreconciliation_tests.cpp new file mode 100644 index 0000000000..bd74998002 --- /dev/null +++ b/src/test/txreconciliation_tests.cpp @@ -0,0 +1,86 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txreconciliation.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(txreconciliation_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(RegisterPeerTest) +{ + TxReconciliationTracker tracker(1); + const uint64_t salt = 0; + + // Prepare a peer for reconciliation. + tracker.PreRegisterPeer(0); + + // Both roles are false, don't register. + BOOST_CHECK(tracker.RegisterPeer(/*peer_id=*/0, /*is_peer_inbound=*/true, + /*is_peer_recon_initiator=*/false, + /*is_peer_recon_responder=*/false, + /*peer_recon_version=*/1, salt) == + ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Invalid roles for the given connection direction. + BOOST_CHECK(tracker.RegisterPeer(0, true, false, true, 1, salt) == ReconciliationRegisterResult::PROTOCOL_VIOLATION); + BOOST_CHECK(tracker.RegisterPeer(0, false, true, false, 1, salt) == ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Invalid version. + BOOST_CHECK(tracker.RegisterPeer(0, true, true, false, 0, salt) == ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Valid registration. + BOOST_REQUIRE(!tracker.IsPeerRegistered(0)); + BOOST_REQUIRE(tracker.RegisterPeer(0, true, true, false, 1, salt) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(0)); + + // Reconciliation version is higher than ours, should be able to register. + BOOST_REQUIRE(!tracker.IsPeerRegistered(1)); + tracker.PreRegisterPeer(1); + BOOST_REQUIRE(tracker.RegisterPeer(1, true, true, false, 2, salt) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(1)); + + // Do not register if there were no pre-registration for the peer. + BOOST_REQUIRE(tracker.RegisterPeer(100, true, true, false, 1, salt) == ReconciliationRegisterResult::NOT_FOUND); + BOOST_CHECK(!tracker.IsPeerRegistered(100)); +} + +BOOST_AUTO_TEST_CASE(ForgetPeerTest) +{ + TxReconciliationTracker tracker(1); + NodeId peer_id0 = 0; + + // Removing peer after pre-registring works and does not let to register the peer. + tracker.PreRegisterPeer(peer_id0); + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(tracker.RegisterPeer(peer_id0, true, true, false, 1, 1) == ReconciliationRegisterResult::NOT_FOUND); + + // Removing peer after it is registered works. + tracker.PreRegisterPeer(peer_id0); + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + BOOST_REQUIRE(tracker.RegisterPeer(peer_id0, true, true, false, 1, 1) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(peer_id0)); + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(!tracker.IsPeerRegistered(peer_id0)); +} + +BOOST_AUTO_TEST_CASE(IsPeerRegisteredTest) +{ + TxReconciliationTracker tracker(1); + NodeId peer_id0 = 0; + + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + tracker.PreRegisterPeer(peer_id0); + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + + BOOST_REQUIRE(tracker.RegisterPeer(peer_id0, true, true, false, 1, 1) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(peer_id0)); + + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(!tracker.IsPeerRegistered(peer_id0)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 2f0021b114..0ca63810f3 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -11,6 +11,7 @@ #include <node/context.h> #include <node/utxo_snapshot.h> #include <rpc/blockchain.h> +#include <test/util/setup_common.h> #include <validation.h> #include <univalue.h> @@ -20,11 +21,24 @@ const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to * malleate the snapshot. + * + * If `reset_chainstate` is true, reset the original chainstate back to the genesis + * block. This allows us to simulate more realistic conditions in which a snapshot is + * loaded into an otherwise mostly-uninitialized datadir. It also allows us to test + * conditions that would otherwise cause shutdowns based on the IBD chainstate going + * past the snapshot it generated. */ template<typename F = decltype(NoMalleation)> static bool -CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F malleation = NoMalleation) +CreateAndActivateUTXOSnapshot( + TestingSetup* fixture, + F malleation = NoMalleation, + bool reset_chainstate = false, + bool in_memory_chainstate = false) { + node::NodeContext& node = fixture->m_node; + fs::path root = fixture->m_path_root; + // Write out a snapshot to the test's tempdir. // int height; @@ -47,7 +61,38 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma malleation(auto_infile, metadata); - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory=*/true); + if (reset_chainstate) { + { + // What follows is code to selectively reset chainstate data without + // disturbing the existing BlockManager instance, which is needed to + // recognize the headers chain previously generated by the chainstate we're + // removing. Without those headers, we can't activate the snapshot below. + // + // This is a stripped-down version of node::LoadChainstate which + // preserves the block index. + LOCK(::cs_main); + uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash(); + node.chainman->ResetChainstates(); + node.chainman->InitializeChainstate(node.mempool.get()); + Chainstate& chain = node.chainman->ActiveChainstate(); + Assert(chain.LoadGenesisBlock()); + // These cache values will be corrected shortly in `MaybeRebalanceCaches`. + chain.InitCoinsDB(1 << 20, true, false, ""); + chain.InitCoinsCache(1 << 20); + chain.CoinsTip().SetBestBlock(gen_hash); + chain.setBlockIndexCandidates.insert(node.chainman->m_blockman.LookupBlockIndex(gen_hash)); + chain.LoadChainTip(); + node.chainman->MaybeRebalanceCaches(); + } + BlockValidationState state; + if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); + } + Assert( + 0 == WITH_LOCK(node.chainman->GetMutex(), return node.chainman->ActiveHeight())); + } + + return node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate); } diff --git a/src/test/util/net.h b/src/test/util/net.h index 73543de4ca..9ae7981980 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -166,6 +166,10 @@ public: return 0; } + bool SetNonBlocking() const override { return true; } + + bool IsSelectable() const override { return true; } + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 74b055ee45..9f29342b10 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -36,6 +36,7 @@ #include <shutdown.h> #include <streams.h> #include <test/util/net.h> +#include <test/util/txmempool.h> #include <timedata.h> #include <txdb.h> #include <txmempool.h> @@ -60,7 +61,6 @@ using node::ApplyArgsManOptions; using node::BlockAssembler; using node::CalculateCacheSizes; using node::LoadChainstate; -using node::NodeContext; using node::RegenerateCommitments; using node::VerifyLoadedChainstate; @@ -162,19 +162,6 @@ BasicTestingSetup::~BasicTestingSetup() gArgs.ClearArgs(); } -CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) -{ - CTxMemPool::Options mempool_opts{ - .estimator = node.fee_estimator.get(), - // Default to always checking mempool regardless of - // chainparams.DefaultConsistencyChecks for tests - .check_ratio = 1, - }; - const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; - Assert(!err); - return mempool_opts; -} - ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : BasicTestingSetup(chainName, extra_args) { @@ -220,17 +207,12 @@ ChainTestingSetup::~ChainTestingSetup() m_node.chainman.reset(); } -TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) - : ChainTestingSetup(chainName, extra_args) +void TestingSetup::LoadVerifyActivateChainstate() { - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - RegisterAllCoreRPCCommands(tableRPC); - node::ChainstateLoadOptions options; options.mempool = Assert(m_node.mempool.get()); - options.block_tree_db_in_memory = true; - options.coins_db_in_memory = true; + options.block_tree_db_in_memory = m_block_tree_db_in_memory; + options.coins_db_in_memory = m_coins_db_in_memory; options.reindex = node::fReindex; options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); options.prune = node::fPruneMode; @@ -246,6 +228,22 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } +} + +TestingSetup::TestingSetup( + const std::string& chainName, + const std::vector<const char*>& extra_args, + const bool coins_db_in_memory, + const bool block_tree_db_in_memory) + : ChainTestingSetup(chainName, extra_args), + m_coins_db_in_memory(coins_db_in_memory), + m_block_tree_db_in_memory(block_tree_db_in_memory) +{ + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + RegisterAllCoreRPCCommands(tableRPC); + + LoadVerifyActivateChainstate(); m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>()); m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman, @@ -263,8 +261,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup(const std::string& chain_name, const std::vector<const char*>& extra_args) - : TestingSetup{chain_name, extra_args} +TestChain100Setup::TestChain100Setup( + const std::string& chain_name, + const std::vector<const char*>& extra_args, + const bool coins_db_in_memory, + const bool block_tree_db_in_memory) + : TestingSetup{CBaseChainParams::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { @@ -423,17 +425,6 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex return mempool_transactions; } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const -{ - return FromTx(MakeTransactionRef(tx)); -} - -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const -{ - return CTxMemPoolEntry(tx, nFee, nTime, nHeight, - spendsCoinbase, sigOpCost, lp); -} - /** * @returns a real block (0000000000013b8ab2cd513b0261a14096412195a72a0c4827d229dcc7e0f7af) * with 9 txs. diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 136ee1fd62..dfa36039a2 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -8,21 +8,23 @@ #include <chainparamsbase.h> #include <fs.h> #include <key.h> -#include <util/system.h> #include <node/caches.h> #include <node/context.h> +#include <primitives/transaction.h> #include <pubkey.h> #include <random.h> #include <stdexcept> -#include <txmempool.h> #include <util/check.h> #include <util/string.h> +#include <util/system.h> #include <util/vector.h> #include <functional> #include <type_traits> #include <vector> +class Chainstate; + /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; @@ -90,9 +92,6 @@ struct BasicTestingSetup { ArgsManager m_args; }; - -CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); - /** Testing setup that performs all steps up until right before * ChainstateManager gets initialized. Meant for testing ChainstateManager * initialization behaviour. @@ -107,7 +106,16 @@ struct ChainTestingSetup : public BasicTestingSetup { /** Testing setup that configures a complete environment. */ struct TestingSetup : public ChainTestingSetup { - explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + bool m_coins_db_in_memory{true}; + bool m_block_tree_db_in_memory{true}; + + void LoadVerifyActivateChainstate(); + + explicit TestingSetup( + const std::string& chainName = CBaseChainParams::MAIN, + const std::vector<const char*>& extra_args = {}, + const bool coins_db_in_memory = true, + const bool block_tree_db_in_memory = true); }; /** Identical to TestingSetup, but chain set to regtest */ @@ -124,8 +132,11 @@ class CScript; * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ struct TestChain100Setup : public TestingSetup { - TestChain100Setup(const std::string& chain_name = CBaseChainParams::REGTEST, - const std::vector<const char*>& extra_args = {}); + TestChain100Setup( + const std::string& chain_name = CBaseChainParams::REGTEST, + const std::vector<const char*>& extra_args = {}, + const bool coins_db_in_memory = true, + const bool block_tree_db_in_memory = true); /** * Create a new block with just given transactions, coinbase paying to @@ -201,33 +212,6 @@ std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseCha return std::make_unique<T>(chain_name, arguments); } -class CTxMemPoolEntry; - -struct TestMemPoolEntryHelper -{ - // Default values - CAmount nFee; - int64_t nTime; - unsigned int nHeight; - bool spendsCoinbase; - unsigned int sigOpCost; - LockPoints lp; - - TestMemPoolEntryHelper() : - nFee(0), nTime(0), nHeight(1), - spendsCoinbase(false), sigOpCost(4) { } - - CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; - CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; - - // Change the default value - TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } - TestMemPoolEntryHelper &Time(int64_t _time) { nTime = _time; return *this; } - TestMemPoolEntryHelper &Height(unsigned int _height) { nHeight = _height; return *this; } - TestMemPoolEntryHelper &SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } - TestMemPoolEntryHelper &SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } -}; - CBlock getBlock13b8a(); // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_* diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp new file mode 100644 index 0000000000..12cc1a4a3d --- /dev/null +++ b/src/test/util/txmempool.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/txmempool.h> + +#include <chainparams.h> +#include <node/context.h> +#include <node/mempool_args.h> +#include <txmempool.h> +#include <util/check.h> +#include <util/time.h> +#include <util/translation.h> + +using node::NodeContext; + +CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) +{ + CTxMemPool::Options mempool_opts{ + .estimator = node.fee_estimator.get(), + // Default to always checking mempool regardless of + // chainparams.DefaultConsistencyChecks for tests + .check_ratio = 1, + }; + const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; + Assert(!err); + return mempool_opts; +} + +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const +{ + return FromTx(MakeTransactionRef(tx)); +} + +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const +{ + return CTxMemPoolEntry(tx, nFee, nTime, nHeight, + spendsCoinbase, sigOpCost, lp); +} diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h new file mode 100644 index 0000000000..70b9ed88db --- /dev/null +++ b/src/test/util/txmempool.h @@ -0,0 +1,37 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_TXMEMPOOL_H +#define BITCOIN_TEST_UTIL_TXMEMPOOL_H + +#include <txmempool.h> + +namespace node { +struct NodeContext; +} + +CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); + +struct TestMemPoolEntryHelper +{ + // Default values + CAmount nFee{0}; + int64_t nTime{0}; + unsigned int nHeight{1}; + bool spendsCoinbase{false}; + unsigned int sigOpCost{4}; + LockPoints lp; + + CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; + CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; + + // Change the default value + TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } + TestMemPoolEntryHelper &Time(int64_t _time) { nTime = _time; return *this; } + TestMemPoolEntryHelper &Height(unsigned int _height) { nHeight = _height; return *this; } + TestMemPoolEntryHelper &SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } + TestMemPoolEntryHelper &SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } +}; + +#endif // BITCOIN_TEST_UTIL_TXMEMPOOL_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 6e59d2e8e6..009c27927f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -26,6 +26,7 @@ #include <util/bitdeque.h> #include <array> +#include <cmath> #include <fstream> #include <limits> #include <map> diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 347a967b33..f868c0d4e6 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -89,7 +89,8 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // After adding some blocks to the tip, best block should have changed. BOOST_CHECK(::g_best_block != curr_tip); - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot( + this, NoMalleation, /*reset_chainstate=*/ true)); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 9fcb7d315a..22b9af1201 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -10,6 +10,7 @@ #include <sync.h> #include <test/util/chainstate.h> #include <test/util/setup_common.h> +#include <timedata.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> @@ -63,7 +64,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // const uint256 snapshot_blockhash = GetRandHash(); - Chainstate& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( + Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot( &mempool, snapshot_blockhash)); chainstates.push_back(&c2); @@ -133,7 +134,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - Chainstate& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); + Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -154,162 +155,240 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -//! Test basic snapshot activation. -BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) -{ - ChainstateManager& chainman = *Assert(m_node.chainman); - - size_t initial_size; - size_t initial_total_coins{100}; - - // Make some initial assertions about the contents of the chainstate. +struct SnapshotTestSetup : TestChain100Setup { + // Run with coinsdb on the filesystem to support, e.g., moving invalidated + // chainstate dirs to "*_invalid". + // + // Note that this means the tests run considerably slower than in-memory DB + // tests, but we can't otherwise test this functionality since it relies on + // destructive filesystem operations. + SnapshotTestSetup() : TestChain100Setup{ + {}, + {}, + /*coins_db_in_memory=*/false, + /*block_tree_db_in_memory=*/false, + } { - LOCK(::cs_main); - CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); - initial_size = ibd_coinscache.GetCacheSize(); - size_t total_coins{0}; - - for (CTransactionRef& txn : m_coinbase_txns) { - COutPoint op{txn->GetHash(), 0}; - BOOST_CHECK(ibd_coinscache.HaveCoin(op)); - total_coins++; - } - - BOOST_CHECK_EQUAL(total_coins, initial_total_coins); - BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } - // Snapshot should refuse to load at this height. - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); - BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); - BOOST_CHECK(!chainman.SnapshotBlockhash()); - - // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can - // be found. - constexpr int snapshot_height = 110; - mineBlocks(10); - initial_size += 10; - initial_total_coins += 10; - - // Should not load malleated snapshots - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // A UTXO is missing but count is correct - metadata.m_coins_count -= 1; - - COutPoint outpoint; - Coin coin; - - auto_infile >> outpoint; - auto_infile >> coin; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Coins count is larger than coins in file - metadata.m_coins_count += 1; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Coins count is smaller than coins in file - metadata.m_coins_count -= 1; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Wrong hash - metadata.m_base_blockhash = uint256::ZERO; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Wrong hash - metadata.m_base_blockhash = uint256::ONE; - })); - - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); - - // Ensure our active chain is the snapshot chainstate. - BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); - BOOST_CHECK_EQUAL( - *chainman.ActiveChainstate().m_from_snapshot_blockhash, - *chainman.SnapshotBlockhash()); - - // Ensure that the genesis block was not marked assumed-valid. - BOOST_CHECK(WITH_LOCK(::cs_main, return !chainman.ActiveChain().Genesis()->IsAssumedValid())); - - const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); - const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); - - BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); - - // To be checked against later when we try loading a subsequent snapshot. - uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; - - // Make some assertions about the both chainstates. These checks ensure the - // legacy chainstate hasn't changed and that the newly created chainstate - // reflects the expected content. + std::tuple<Chainstate*, Chainstate*> SetupSnapshot() { - LOCK(::cs_main); - int chains_tested{0}; + ChainstateManager& chainman = *Assert(m_node.chainman); - for (Chainstate* chainstate : chainman.GetAll()) { - BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); - CCoinsViewCache& coinscache = chainstate->CoinsTip(); + BOOST_CHECK(!chainman.IsSnapshotActive()); - // Both caches will be empty initially. - BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + { + LOCK(::cs_main); + BOOST_CHECK(!chainman.IsSnapshotValidated()); + BOOST_CHECK(!node::FindSnapshotChainstateDir()); + } + size_t initial_size; + size_t initial_total_coins{100}; + + // Make some initial assertions about the contents of the chainstate. + { + LOCK(::cs_main); + CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); + initial_size = ibd_coinscache.GetCacheSize(); size_t total_coins{0}; for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; - BOOST_CHECK(coinscache.HaveCoin(op)); + BOOST_CHECK(ibd_coinscache.HaveCoin(op)); total_coins++; } - BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); BOOST_CHECK_EQUAL(total_coins, initial_total_coins); - chains_tested++; + BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } - BOOST_CHECK_EQUAL(chains_tested, 2); - } + Chainstate& validation_chainstate = chainman.ActiveChainstate(); + + // Snapshot should refuse to load at this height. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); + BOOST_CHECK(!chainman.SnapshotBlockhash()); + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + constexpr int snapshot_height = 110; + mineBlocks(10); + initial_size += 10; + initial_total_coins += 10; + + // Should not load malleated snapshots + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // A UTXO is missing but count is correct + metadata.m_coins_count -= 1; + + COutPoint outpoint; + Coin coin; + + auto_infile >> outpoint; + auto_infile >> coin; + })); + + BOOST_CHECK(!node::FindSnapshotChainstateDir()); + + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is larger than coins in file + metadata.m_coins_count += 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is smaller than coins in file + metadata.m_coins_count -= 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Wrong hash + metadata.m_base_blockhash = uint256::ZERO; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Wrong hash + metadata.m_base_blockhash = uint256::ONE; + })); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); + BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); + BOOST_CHECK_EQUAL( + *chainman.ActiveChainstate().m_from_snapshot_blockhash, + *chainman.SnapshotBlockhash()); + + Chainstate& snapshot_chainstate = chainman.ActiveChainstate(); + + { + LOCK(::cs_main); - // Mine some new blocks on top of the activated snapshot chainstate. - constexpr size_t new_coins{100}; - mineBlocks(new_coins); // Defined in TestChain100Setup. + fs::path found = *node::FindSnapshotChainstateDir(); - { - LOCK(::cs_main); - size_t coins_in_active{0}; - size_t coins_in_background{0}; - size_t coins_missing_from_background{0}; + // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. + BOOST_CHECK_EQUAL( + *node::ReadSnapshotBaseBlockhash(found), + *chainman.SnapshotBlockhash()); - for (Chainstate* chainstate : chainman.GetAll()) { - BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); - CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_background = chainstate != &chainman.ActiveChainstate(); + // Ensure that the genesis block was not marked assumed-valid. + BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); + } - for (CTransactionRef& txn : m_coinbase_txns) { - COutPoint op{txn->GetHash(), 0}; - if (coinscache.HaveCoin(op)) { - (is_background ? coins_in_background : coins_in_active)++; - } else if (is_background) { - coins_missing_from_background++; + const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); + const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); + + BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); + + // To be checked against later when we try loading a subsequent snapshot. + uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; + + // Make some assertions about the both chainstates. These checks ensure the + // legacy chainstate hasn't changed and that the newly created chainstate + // reflects the expected content. + { + LOCK(::cs_main); + int chains_tested{0}; + + for (Chainstate* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + + // Both caches will be empty initially. + BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(coinscache.HaveCoin(op)); + total_coins++; } + + BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + chains_tested++; } + + BOOST_CHECK_EQUAL(chains_tested, 2); } - BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); + // Mine some new blocks on top of the activated snapshot chainstate. + constexpr size_t new_coins{100}; + mineBlocks(new_coins); // Defined in TestChain100Setup. + + { + LOCK(::cs_main); + size_t coins_in_active{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; + + for (Chainstate* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + bool is_background = chainstate != &chainman.ActiveChainstate(); + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + if (coinscache.HaveCoin(op)) { + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; + } + } + } + + BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); + } + + // Snapshot should refuse to load after one has already loaded. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); + + // Snapshot blockhash should be unchanged. + BOOST_CHECK_EQUAL( + *chainman.ActiveChainstate().m_from_snapshot_blockhash, + loaded_snapshot_blockhash); + return std::make_tuple(&validation_chainstate, &snapshot_chainstate); } - // Snapshot should refuse to load after one has already loaded. - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + // Simulate a restart of the node by flushing all state to disk, clearing the + // existing ChainstateManager, and unloading the block index. + // + // @returns a reference to the "restarted" ChainstateManager + ChainstateManager& SimulateNodeRestart() + { + ChainstateManager& chainman = *Assert(m_node.chainman); + + BOOST_TEST_MESSAGE("Simulating node restart"); + { + LOCK(::cs_main); + for (Chainstate* cs : chainman.GetAll()) { + cs->ForceFlushStateToDisk(); + } + chainman.ResetChainstates(); + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); + const ChainstateManager::Options chainman_opts{ + .chainparams = ::Params(), + .adjusted_time_callback = GetAdjustedTime, + }; + // For robustness, ensure the old manager is destroyed before creating a + // new one. + m_node.chainman.reset(); + m_node.chainman.reset(new ChainstateManager(chainman_opts)); + } + return *Assert(m_node.chainman); + } +}; - // Snapshot blockhash should be unchanged. - BOOST_CHECK_EQUAL( - *chainman.ActiveChainstate().m_from_snapshot_blockhash, - loaded_snapshot_blockhash); +//! Test basic snapshot activation. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) +{ + this->SetupSnapshot(); } //! Test LoadBlockIndex behavior when multiple chainstates are in use. @@ -374,7 +453,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); Chainstate& cs2 = WITH_LOCK(::cs_main, - return chainman.InitializeChainstate(&mempool, GetRandHash())); + return chainman.ActivateExistingSnapshot(&mempool, GetRandHash())); reload_all_block_indexes(); @@ -390,4 +469,59 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); } +//! Ensure that snapshot chainstates initialize properly when found on disk. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) +{ + this->SetupSnapshot(); + + ChainstateManager& chainman = *Assert(m_node.chainman); + + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); + BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); + + BOOST_CHECK(chainman.IsSnapshotActive()); + const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(), + return chainman.ActiveTip()->GetBlockHash()); + + auto all_chainstates = chainman.GetAll(); + BOOST_CHECK_EQUAL(all_chainstates.size(), 2); + + // Test that simulating a shutdown (resetting ChainstateManager) and then performing + // chainstate reinitializing successfully cleans up the background-validation + // chainstate data, and we end up with a single chainstate that is at tip. + ChainstateManager& chainman_restarted = this->SimulateNodeRestart(); + + BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); + + // This call reinitializes the chainstates. + this->LoadVerifyActivateChainstate(); + + { + LOCK(chainman_restarted.GetMutex()); + BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 2); + BOOST_CHECK(chainman_restarted.IsSnapshotActive()); + BOOST_CHECK(!chainman_restarted.IsSnapshotValidated()); + + BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); + BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); + } + + BOOST_TEST_MESSAGE( + "Ensure we can mine blocks on top of the initialized snapshot chainstate"); + mineBlocks(10); + { + LOCK(chainman_restarted.GetMutex()); + BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); + + // Background chainstate should be unaware of new blocks on the snapshot + // chainstate. + for (Chainstate* cs : chainman_restarted.GetAll()) { + if (cs != &chainman_restarted.ActiveChainstate()) { + BOOST_CHECK_EQUAL(cs->m_chain.Height(), 110); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.h b/src/txdb.h index a04596f7bb..8c41e26f6a 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -9,6 +9,7 @@ #include <coins.h> #include <dbwrapper.h> #include <sync.h> +#include <fs.h> #include <memory> #include <optional> @@ -72,6 +73,9 @@ public: //! Dynamically alter the underlying leveldb cache size. void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! @returns filesystem path to on-disk storage or std::nullopt if in memory. + std::optional<fs::path> StoragePath() { return m_db->StoragePath(); } }; /** Access to the block database (blocks/index/) */ diff --git a/src/util/error.cpp b/src/util/error.cpp index 33a35a6d59..390cb6c11b 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -49,6 +49,11 @@ bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBi return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); } +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) +{ + return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); +} + bilingual_str AmountHighWarn(const std::string& optname) { return strprintf(_("%s is set very high!"), optname); diff --git a/src/util/error.h b/src/util/error.h index 0429de651a..27916501f0 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -39,6 +39,8 @@ bilingual_str TransactionErrorString(const TransactionError error); bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); + bilingual_str AmountHighWarn(const std::string& optname); bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 125dbc7f18..84ac2759fa 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -117,6 +117,34 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } +bool Sock::SetNonBlocking() const +{ +#ifdef WIN32 + u_long on{1}; + if (ioctlsocket(m_socket, FIONBIO, &on) == SOCKET_ERROR) { + return false; + } +#else + const int flags{fcntl(m_socket, F_GETFL, 0)}; + if (flags == SOCKET_ERROR) { + return false; + } + if (fcntl(m_socket, F_SETFL, flags | O_NONBLOCK) == SOCKET_ERROR) { + return false; + } +#endif + return true; +} + +bool Sock::IsSelectable() const +{ +#if defined(USE_POLL) || defined(WIN32) + return true; +#else + return m_socket < FD_SETSIZE; +#endif +} + bool Sock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { // We need a `shared_ptr` owning `this` for `WaitMany()`, but don't want @@ -185,10 +213,10 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per SOCKET socket_max{0}; for (const auto& [sock, events] : events_per_sock) { - const auto& s = sock->m_socket; - if (!IsSelectableSocket(s)) { + if (!sock->IsSelectable()) { return false; } + const auto& s = sock->m_socket; if (events.requested & RECV) { FD_SET(s, &recv); } diff --git a/src/util/sock.h b/src/util/sock.h index 38a7dc80d6..7912284904 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -133,6 +133,18 @@ public: */ [[nodiscard]] virtual int GetSockName(sockaddr* name, socklen_t* name_len) const; + /** + * Set the non-blocking option on the socket. + * @return true if set successfully + */ + [[nodiscard]] virtual bool SetNonBlocking() const; + + /** + * Check if the underlying socket can be used for `select(2)` (or the `Wait()` method). + * @return true if selectable + */ + [[nodiscard]] virtual bool IsSelectable() const; + using Event = uint8_t; /** diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index b5ac151374..e28ca8e73a 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -97,8 +97,9 @@ std::vector<Byte> ParseHex(std::string_view str) template std::vector<std::byte> ParseHex(std::string_view); template std::vector<uint8_t> ParseHex(std::string_view); -void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) +bool SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) { + bool valid = false; size_t colon = in.find_last_of(':'); // if a : is found, and it either follows a [...], or no other : is in the string, treat it as port separator bool fHaveColon = colon != in.npos; @@ -109,13 +110,18 @@ void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) if (ParseUInt16(in.substr(colon + 1), &n)) { in = in.substr(0, colon); portOut = n; + valid = (portOut != 0); } + } else { + valid = true; } if (in.size() > 0 && in[0] == '[' && in[in.size() - 1] == ']') { hostOut = in.substr(1, in.size() - 2); } else { hostOut = in; } + + return valid; } std::string EncodeBase64(Span<const unsigned char> input) diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 14867b21b2..94bc6cc2f3 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -88,7 +88,16 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); */ std::string EncodeBase32(std::string_view str, bool pad = true); -void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); +/** + * Splits socket address string into host string and port value. + * Validates port value. + * + * @param[in] in The socket address string to split. + * @param[out] portOut Port-portion of the input, if found and parsable. + * @param[out] hostOut Host-portion of the input, if found. + * @return true if port-portion is absent or within its allowed range, otherwise false + */ +bool SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); // LocaleIndependentAtoi is provided for backwards compatibility reasons. // diff --git a/src/validation.cpp b/src/validation.cpp index d7ecbbec84..37e68cfe4a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1512,7 +1512,7 @@ void Chainstate::InitCoinsDB( fs::path leveldb_name) { if (m_from_snapshot_blockhash) { - leveldb_name += "_" + m_from_snapshot_blockhash->ToString(); + leveldb_name += node::SNAPSHOT_CHAINSTATE_SUFFIX; } m_coins_views = std::make_unique<CoinsViews>( @@ -1837,11 +1837,21 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn return DISCONNECT_FAILED; } + // Ignore blocks that contain transactions which are 'overwritten' by later transactions, + // unless those are already completely spent. + // See https://github.com/bitcoin/bitcoin/issues/22596 for additional information. + // Note: the blocks specified here are different than the ones used in ConnectBlock because DisconnectBlock + // unwinds the blocks in reverse. As a result, the inconsistency is not discovered until the earlier + // blocks with the duplicate coinbase transactions are disconnected. + bool fEnforceBIP30 = !((pindex->nHeight==91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight==91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))); + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); bool is_coinbase = tx.IsCoinBase(); + bool is_bip30_exception = (is_coinbase && !fEnforceBIP30); // Check that all outputs are available and match the outputs in the block itself // exactly. @@ -1851,7 +1861,9 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn Coin coin; bool is_spent = view.SpendCoin(out, &coin); if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { - fClean = false; // transaction output mismatch + if (!is_bip30_exception) { + fClean = false; // transaction output mismatch + } } } } @@ -4483,7 +4495,18 @@ void Chainstate::LoadExternalBlockFile( } } } catch (const std::exception& e) { - LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + // historical bugs added extra data to the block files that does not deserialize cleanly. + // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process. + // the code that reads the block files deals with invalid data by simply ignoring it. + // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly + // and passes all of the other block validation checks dealing with POW and the merkle root, etc... + // we merely note with this informational log message when unexpected data is encountered. + // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but + // less likely scenarios. we don't have enough information to tell a difference here. + // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator + // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and + // perhaps ordered, block files for later reindexing. + LogPrint(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what()); } } } catch (const std::runtime_error& e) { @@ -4800,28 +4823,15 @@ std::vector<Chainstate*> ChainstateManager::GetAll() return out; } -Chainstate& ChainstateManager::InitializeChainstate( - CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash) +Chainstate& ChainstateManager::InitializeChainstate(CTxMemPool* mempool) { AssertLockHeld(::cs_main); - bool is_snapshot = snapshot_blockhash.has_value(); - std::unique_ptr<Chainstate>& to_modify = - is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; + assert(!m_ibd_chainstate); + assert(!m_active_chainstate); - if (to_modify) { - throw std::logic_error("should not be overwriting a chainstate"); - } - to_modify.reset(new Chainstate(mempool, m_blockman, *this, snapshot_blockhash)); - - // Snapshot chainstates and initial IBD chaintates always become active. - if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { - LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); - m_active_chainstate = to_modify.get(); - } else { - throw std::logic_error("unexpected chainstate activation"); - } - - return *to_modify; + m_ibd_chainstate = std::make_unique<Chainstate>(mempool, m_blockman, *this); + m_active_chainstate = m_ibd_chainstate.get(); + return *m_active_chainstate; } const AssumeutxoData* ExpectedAssumeutxo( @@ -4836,6 +4846,46 @@ const AssumeutxoData* ExpectedAssumeutxo( return nullptr; } +static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + AssertLockHeld(::cs_main); + + if (is_snapshot) { + fs::path base_blockhash_path = db_path / node::SNAPSHOT_BLOCKHASH_FILENAME; + + if (fs::exists(base_blockhash_path)) { + bool removed = fs::remove(base_blockhash_path); + if (!removed) { + LogPrintf("[snapshot] failed to remove file %s\n", + fs::PathToString(base_blockhash_path)); + } + } else { + LogPrintf("[snapshot] snapshot chainstate dir being removed lacks %s file\n", + fs::PathToString(node::SNAPSHOT_BLOCKHASH_FILENAME)); + } + } + + std::string path_str = fs::PathToString(db_path); + LogPrintf("Removing leveldb dir at %s\n", path_str); + + // We have to destruct before this call leveldb::DB in order to release the db + // lock, otherwise `DestroyDB` will fail. See `leveldb::~DBImpl()`. + const bool destroyed = dbwrapper::DestroyDB(path_str, {}).ok(); + + if (!destroyed) { + LogPrintf("error: leveldb DestroyDB call failed on %s\n", path_str); + } + + // Datadir should be removed from filesystem; otherwise initialization may detect + // it on subsequent statups and get confused. + // + // If the base_blockhash_path removal above fails in the case of snapshot + // chainstates, this will return false since leveldb won't remove a non-empty + // directory. + return destroyed && !fs::exists(db_path); +} + bool ChainstateManager::ActivateSnapshot( AutoFile& coins_file, const SnapshotMetadata& metadata, @@ -4893,11 +4943,34 @@ bool ChainstateManager::ActivateSnapshot( static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC)); } - const bool snapshot_ok = this->PopulateAndValidateSnapshot( + bool snapshot_ok = this->PopulateAndValidateSnapshot( *snapshot_chainstate, coins_file, metadata); + // If not in-memory, persist the base blockhash for use during subsequent + // initialization. + if (!in_memory) { + LOCK(::cs_main); + if (!node::WriteSnapshotBaseBlockhash(*snapshot_chainstate)) { + snapshot_ok = false; + } + } if (!snapshot_ok) { - WITH_LOCK(::cs_main, this->MaybeRebalanceCaches()); + LOCK(::cs_main); + this->MaybeRebalanceCaches(); + + // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir + // has been created, so only attempt removal if we got that far. + if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) { + // We have to destruct leveldb::DB in order to release the db lock, otherwise + // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`. + // Destructing the chainstate (and so resetting the coinsviews object) does this. + snapshot_chainstate.reset(); + bool removed = DeleteCoinsDBFromDisk(*snapshot_datadir, /*is_snapshot=*/true); + if (!removed) { + AbortNode(strprintf("Failed to remove snapshot chainstate dir (%s). " + "Manually remove it before restarting.\n", fs::PathToString(*snapshot_datadir))); + } + } return false; } @@ -5171,6 +5244,13 @@ void ChainstateManager::MaybeRebalanceCaches() } } +void ChainstateManager::ResetChainstates() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; +} + ChainstateManager::~ChainstateManager() { LOCK(::cs_main); @@ -5182,3 +5262,31 @@ ChainstateManager::~ChainstateManager() i.clear(); } } + +bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) +{ + assert(!m_snapshot_chainstate); + std::optional<fs::path> path = node::FindSnapshotChainstateDir(); + if (!path) { + return false; + } + std::optional<uint256> base_blockhash = node::ReadSnapshotBaseBlockhash(*path); + if (!base_blockhash) { + return false; + } + LogPrintf("[snapshot] detected active snapshot chainstate (%s) - loading\n", + fs::PathToString(*path)); + + this->ActivateExistingSnapshot(mempool, *base_blockhash); + return true; +} + +Chainstate& ChainstateManager::ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) +{ + assert(!m_snapshot_chainstate); + m_snapshot_chainstate = + std::make_unique<Chainstate>(mempool, m_blockman, *this, base_blockhash); + LogPrintf("[snapshot] switching active chainstate to %s\n", m_snapshot_chainstate->ToString()); + m_active_chainstate = m_snapshot_chainstate.get(); + return *m_snapshot_chainstate; +} diff --git a/src/validation.h b/src/validation.h index 9e60292beb..fb6d59f92e 100644 --- a/src/validation.h +++ b/src/validation.h @@ -922,17 +922,11 @@ public: //! coins databases. This will be split somehow across chainstates. int64_t m_total_coinsdb_cache{0}; - //! Instantiate a new chainstate and assign it based upon whether it is - //! from a snapshot. + //! Instantiate a new chainstate. //! //! @param[in] mempool The mempool to pass to the chainstate // constructor - //! @param[in] snapshot_blockhash If given, signify that this chainstate - //! is based on a snapshot. - Chainstate& InitializeChainstate( - CTxMemPool* mempool, - const std::optional<uint256>& snapshot_blockhash = std::nullopt) - LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + Chainstate& InitializeChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. std::vector<Chainstate*> GetAll(); @@ -1046,6 +1040,17 @@ public: * information. */ void ReportHeadersPresync(const arith_uint256& work, int64_t height, int64_t timestamp); + //! When starting up, search the datadir for a chainstate based on a UTXO + //! snapshot that is in the process of being validated. + bool DetectSnapshotChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + void ResetChainstates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Switch the active chainstate to one based on a UTXO snapshot that was loaded + //! previously. + Chainstate& ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + ~ChainstateManager(); }; diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 8babbb4298..1206a428fc 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -293,10 +293,7 @@ RPCHelpMan importaddress() if (fRescan) { RescanWallet(*pwallet, reserver); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); } return UniValue::VNULL; @@ -474,10 +471,7 @@ RPCHelpMan importpubkey() if (fRescan) { RescanWallet(*pwallet, reserver); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); } return UniValue::VNULL; @@ -1406,10 +1400,7 @@ RPCHelpMan importmulti() } if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); @@ -1700,10 +1691,7 @@ RPCHelpMan importdescriptors() // Rescan the blockchain using the lowest timestamp if (rescan) { int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 6da0741d59..39afb79600 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -28,6 +28,9 @@ util::Result<CTxDestination> LegacyScriptPubKeyMan::GetNewDestination(const Outp } assert(type != OutputType::BECH32M); + // Fill-up keypool if needed + TopUp(); + LOCK(cs_KeyStore); // Generate a new key that is added to wallet @@ -304,6 +307,9 @@ util::Result<CTxDestination> LegacyScriptPubKeyMan::GetReservedDestination(const return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } + // Fill-up keypool if needed + TopUp(); + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } @@ -1983,7 +1989,7 @@ util::Result<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const std::optional<OutputType> desc_addr_type = m_wallet_descriptor.descriptor->GetOutputType(); assert(desc_addr_type); if (type != *desc_addr_type) { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); } TopUp(); @@ -2001,11 +2007,8 @@ util::Result<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const } CTxDestination dest; - std::optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); - if (out_script_type && out_script_type == type) { - ExtractDestination(scripts_temp[0], dest); - } else { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + if (!ExtractDestination(scripts_temp[0], dest)) { + return util::Error{_("Error: Cannot extract destination from the generated scriptpubkey")}; // shouldn't happen } m_wallet_descriptor.next_index++; WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); diff --git a/src/wallet/test/feebumper_tests.cpp b/src/wallet/test/feebumper_tests.cpp index 6add86dc3d..2480a9b4e1 100644 --- a/src/wallet/test/feebumper_tests.cpp +++ b/src/wallet/test/feebumper_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php. +#include <consensus/validation.h> +#include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> #include <util/strencodings.h> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 461e64751a..671c432b10 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -21,6 +21,7 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> +#include <random.h> #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> @@ -1903,11 +1904,29 @@ std::set<uint256> CWallet::GetTxConflicts(const CWalletTx& wtx) const return result; } +bool CWallet::ShouldResend() const +{ + // Don't attempt to resubmit if the wallet is configured to not broadcast + if (!fBroadcastTransactions) return false; + + // During reindex, importing and IBD, old wallet transactions become + // unconfirmed. Don't resend them as that would spam other nodes. + // We only allow forcing mempool submission when not relaying to avoid this spam. + if (!chain().isReadyToBroadcast()) return false; + + // Do this infrequently and randomly to avoid giving away + // that these are our transactions. + if (GetTime() < m_next_resend) return false; + + return true; +} + +int64_t CWallet::GetDefaultNextResend() { return GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); } + // Resubmit transactions from the wallet to the mempool, optionally asking the // mempool to relay them. On startup, we will do this for all unconfirmed // transactions but will not ask the mempool to relay them. We do this on startup -// to ensure that our own mempool is aware of our transactions, and to also -// initialize m_next_resend so that the actual rebroadcast is scheduled. There +// to ensure that our own mempool is aware of our transactions. There // is a privacy side effect here as not broadcasting on startup also means that we won't // inform the world of our wallet's state, particularly if the wallet (or node) is not // yet synced. @@ -1934,17 +1953,6 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) // even if forcing. if (!fBroadcastTransactions) return; - // During reindex, importing and IBD, old wallet transactions become - // unconfirmed. Don't resend them as that would spam other nodes. - // We only allow forcing mempool submission when not relaying to avoid this spam. - if (!force && relay && !chain().isReadyToBroadcast()) return; - - // Do this infrequently and randomly to avoid giving away - // that these are our transactions. - if (!force && GetTime() < m_next_resend) return; - // resend 12-36 hours from now, ~1 day on average. - m_next_resend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); - int submitted_tx_count = 0; { // cs_wallet scope @@ -1957,7 +1965,7 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) // Only rebroadcast unconfirmed txs if (!wtx.isUnconfirmed()) continue; - // attempt to rebroadcast all txes more than 5 minutes older than + // Attempt to rebroadcast all txes more than 5 minutes older than // the last block, or all txs if forcing. if (!force && wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; to_submit.insert(&wtx); @@ -1979,7 +1987,9 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) void MaybeResendWalletTxs(WalletContext& context) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { + if (!pwallet->ShouldResend()) continue; pwallet->ResubmitWalletTransactions(/*relay=*/true, /*force=*/false); + pwallet->SetNextResend(); } } @@ -2376,7 +2386,6 @@ util::Result<CTxDestination> CWallet::GetNewDestination(const OutputType type, c return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - spk_man->TopUp(); auto op_dest = spk_man->GetNewDestination(type); if (op_dest) { SetAddressBook(*op_dest, label, "receive"); @@ -2470,10 +2479,7 @@ util::Result<CTxDestination> ReserveDestination::GetReservedDestination(bool int return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - if (nIndex == -1) - { - m_spk_man->TopUp(); - + if (nIndex == -1) { CKeyPool keypool; auto op_address = m_spk_man->GetReservedDestination(type, internal, nIndex, keypool); if (!op_address) return op_address; @@ -3198,14 +3204,12 @@ bool CWallet::UpgradeWallet(int version, bilingual_str& error) void CWallet::postInitProcess() { - LOCK(cs_wallet); - // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); // Update wallet transactions with current mempool transactions. - chain().requestMempoolTransactions(*this); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); } bool CWallet::BackupWallet(const std::string& strDest) const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 953d3e5bde..352f43ef99 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -250,7 +250,7 @@ private: int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; /** The next scheduled rebroadcast of wallet transactions. */ - std::atomic<int64_t> m_next_resend{}; + int64_t m_next_resend{GetDefaultNextResend()}; /** Whether this wallet will submit newly created transactions to the node's mempool and * prompt rebroadcasts (see ResendWalletTransactions()). */ bool fBroadcastTransactions = false; @@ -348,6 +348,8 @@ private: */ static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings); + static int64_t GetDefaultNextResend(); + public: /** * Main wallet lock. @@ -399,7 +401,6 @@ public: TxItems wtxOrdered; int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; - uint64_t nAccountingEntryNumber = 0; std::map<CTxDestination, CAddressBookData> m_address_book GUARDED_BY(cs_wallet); const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -537,6 +538,10 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress); void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; + /** Set the next time this wallet should resend transactions to 12-36 hours from now, ~1 day on average. */ + void SetNextResend() { m_next_resend = GetDefaultNextResend(); } + /** Return true if all conditions for periodically resending transactions are met. */ + bool ShouldResend() const; void ResubmitWalletTransactions(bool relay, bool force); OutputType TransactionChangeType(const std::optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) const; diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index fa3944e32b..97c2599366 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H - +#include <cstdint> #include <memory> #include <string> diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index b9b7019a3c..6ee134f392 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -3,13 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <zmq/zmqnotificationinterface.h> + +#include <logging.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <util/system.h> +#include <validationinterface.h> +#include <zmq/zmqabstractnotifier.h> #include <zmq/zmqpublishnotifier.h> #include <zmq/zmqutil.h> #include <zmq.h> -#include <primitives/block.h> -#include <util/system.h> +#include <cassert> +#include <map> +#include <string> +#include <utility> +#include <vector> CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(nullptr) { diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 8f81bfd63f..585e900ca6 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -5,10 +5,14 @@ #ifndef BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H #define BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H +#include <primitives/transaction.h> #include <validationinterface.h> + +#include <cstdint> #include <list> #include <memory> +class CBlock; class CBlockIndex; class CZMQAbstractNotifier; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 51c8ad515e..eaf3455296 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -6,21 +6,37 @@ #include <chain.h> #include <chainparams.h> +#include <crypto/common.h> +#include <logging.h> +#include <netaddress.h> #include <netbase.h> #include <node/blockstorage.h> +#include <primitives/block.h> +#include <primitives/transaction.h> #include <rpc/server.h> +#include <serialize.h> #include <streams.h> -#include <util/system.h> +#include <sync.h> +#include <uint256.h> +#include <version.h> #include <zmq/zmqutil.h> #include <zmq.h> +#include <cassert> #include <cstdarg> #include <cstddef> +#include <cstdint> +#include <cstring> #include <map> #include <optional> #include <string> #include <utility> +#include <vector> + +namespace Consensus { +struct Params; +} using node::ReadBlockFromDisk; diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index c1d66bddb1..fcedd1aabe 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -7,7 +7,11 @@ #include <zmq/zmqabstractnotifier.h> +#include <cstddef> +#include <cstdint> + class CBlockIndex; +class CTransaction; class CZMQAbstractPublishNotifier : public CZMQAbstractNotifier { diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index ec6d1cbba3..047e6bf9b7 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -11,6 +11,11 @@ #include <univalue.h> +#include <list> +#include <string> + +class JSONRPCRequest; + namespace { static RPCHelpMan getzmqnotifications() diff --git a/test/functional/data/rpc_psbt.json b/test/functional/data/rpc_psbt.json index 657faebffc..3127350872 100644 --- a/test/functional/data/rpc_psbt.json +++ b/test/functional/data/rpc_psbt.json @@ -44,6 +44,10 @@ [ "cHNidP8BAKOro2MDAwMDA5ggCAAA////CQAtAAD+///1AAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAD+///1Zm9ybmV3nWx1Y2vmelLmegAAAAAAAAAAAAAAAAAAAAMKAwMDAwMDAwMDAwMACvMBA3FkAAAAAAAAAAAABAAlAAAAAAAAACEWDQ0zDQ0NDQ0NDQ0NCwEAAH9/f39/fwMAAABNo6P///kAAA==", "Input Taproot BIP32 keypath has an invalid length" + ], + [ + "cHNidP8BAIkCAAAAAapfm08b0MipBvW9thL06f8rMbeazW7TIa0W9plHj4WoAAAAAAD9////AoCWmAAAAAAAIlEgC+blBlIP1iijRWxqjw1u9H02sqr7y8fno6/LdnvGqPl895x2AAAAACJRIM5wyjSexMbADl4K+AI1/68zyaDlE7guKvrEDUAjwqU1AAAAAAABASsAlDV3AAAAACJRIDfCpO/CIAqc0JKgBhsCfaPGdyroYtmH+4gQK/Mnn72UIRZGOixxmh9h2gqDIecYHcQHRa8w+Sokc//iDiqXz7uMGRkAHzYIzlYAAIABAACAAAAAgAAAAABhAAAAARcgRjoscZofYdoKgyHnGB3EB0WvMPkqJHP/4g4ql8+7jBkAAQUg1YCB33LpmkGemw3ncz7fcnjhL/bBG/PjH8vpgr2L3cUBBgAhB9WAgd9y6ZpBnpsN53M+33J44S/2wRvz4x/L6YK9i93FGQAfNgjOVgAAgAEAAIAAAACAAAAAAGIAAAAAAQUg9jMNus8cd+GAosBk9wn+pNP9wn7A+jy2Vq0cy+siJ8wBBgAhB/YzDbrPHHfhgKLAZPcJ/qTT/cJ+wPo8tlatHMvrIifMGQAfNgjOVgAAgAEAAIAAAACAAQAAAFEBAAAA", + "Output Taproot tree must not be empty" ] ], "valid" : [ diff --git a/test/functional/feature_discover.py b/test/functional/feature_discover.py new file mode 100755 index 0000000000..7f4b81114e --- /dev/null +++ b/test/functional/feature_discover.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test -discover command.""" + +import socket + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +def is_valid_ipv4_address(address): + try: + socket.inet_aton(address) + except socket.error: + return False + return True + + +def is_valid_ipv6_address(address): + try: + socket.inet_pton(socket.AF_INET6, address) + except socket.error: + return False + return True + + +class DiscoverTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.bind_to_localhost_only = False + self.num_nodes = 1 + + def validate_addresses(self, addresses_obj): + for address_obj in addresses_obj: + address = address_obj['address'] + self.log.info(f"Validating {address}") + valid = (is_valid_ipv4_address(address) + or is_valid_ipv6_address(address)) + assert_equal(valid, True) + + def test_local_addresses(self, test_case, *, expect_empty=False): + self.log.info(f"Restart node with {test_case}") + self.restart_node(0, test_case) + network_info = self.nodes[0].getnetworkinfo() + network_enabled = [n for n in network_info['networks'] + if n['reachable'] and n['name'] in ['ipv4', 'ipv6']] + local_addrs = list(network_info["localaddresses"]) + if expect_empty or not network_enabled: + assert_equal(local_addrs, []) + elif len(local_addrs) > 0: + self.validate_addresses(local_addrs) + + def run_test(self): + test_cases = [ + ["-listen", "-discover"], + ["-discover"], + ] + + test_cases_empty = [ + ["-discover=0"], + ["-listen", "-discover=0"], + [], + ] + + for test_case in test_cases: + self.test_local_addresses(test_case, expect_empty=False) + + for test_case in test_cases_empty: + self.test_local_addresses(test_case, expect_empty=True) + + +if __name__ == '__main__': + DiscoverTest().main() diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 13c7326519..56d093c396 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -55,7 +55,6 @@ class InitStressTest(BitcoinTestFramework): b'Loading P2P addresses', b'Loading banlist', b'Loading block index', - b'Switching active chainstate', b'Checking all blk files are present', b'Loaded best chain:', b'init message: Verifying blocks', diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index c90852562e..18b079cd71 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -317,19 +317,34 @@ class ProxyTest(BitcoinTestFramework): self.stop_node(1) - self.log.info("Test passing invalid -proxy raises expected init error") - self.nodes[1].extra_args = ["-proxy=abc:def"] - msg = "Error: Invalid -proxy address or hostname: 'abc:def'" + self.log.info("Test passing invalid -proxy hostname raises expected init error") + self.nodes[1].extra_args = ["-proxy=abc..abc:23456"] + msg = "Error: Invalid -proxy address or hostname: 'abc..abc:23456'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) - self.log.info("Test passing invalid -onion raises expected init error") - self.nodes[1].extra_args = ["-onion=xyz:abc"] - msg = "Error: Invalid -onion address or hostname: 'xyz:abc'" + self.log.info("Test passing invalid -proxy port raises expected init error") + self.nodes[1].extra_args = ["-proxy=192.0.0.1:def"] + msg = "Error: Invalid port specified in -proxy: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) - self.log.info("Test passing invalid -i2psam raises expected init error") - self.nodes[1].extra_args = ["-i2psam=def:xyz"] - msg = "Error: Invalid -i2psam address or hostname: 'def:xyz'" + self.log.info("Test passing invalid -onion hostname raises expected init error") + self.nodes[1].extra_args = ["-onion=xyz..xyz:23456"] + msg = "Error: Invalid -onion address or hostname: 'xyz..xyz:23456'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -onion port raises expected init error") + self.nodes[1].extra_args = ["-onion=192.0.0.1:def"] + msg = "Error: Invalid port specified in -onion: '192.0.0.1:def'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam hostname raises expected init error") + self.nodes[1].extra_args = ["-i2psam=def..def:23456"] + msg = "Error: Invalid -i2psam address or hostname: 'def..def:23456'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam port raises expected init error") + self.nodes[1].extra_args = ["-i2psam=192.0.0.1:def"] + msg = "Error: Invalid port specified in -i2psam: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -onlynet=i2p without -i2psam raises expected init error") diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 7dbeccbc09..4126ffcd51 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -358,6 +358,8 @@ class PruneTest(BitcoinTestFramework): self.restart_node(2, extra_args=["-prune=550"]) self.log.info("Success") + assert_raises_rpc_error(-4, "Importing wallets is disabled when blocks are pruned", self.nodes[2].importwallet, "abc") + # check that wallet loads successfully when restarting a pruned node after IBD. # this was reported to fail in #7494. self.log.info("Syncing node 5 to test wallet") diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 610a28c56b..24252610be 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -387,6 +387,17 @@ class RESTTest (BitcoinTestFramework): assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")) + self.log.info("Test the /deploymentinfo URI") + + deployment_info = self.nodes[0].getdeploymentinfo() + assert_equal(deployment_info, self.test_rest_request('/deploymentinfo')) + + non_existing_blockhash = '42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4' + resp = self.test_rest_request(f'/deploymentinfo/{non_existing_blockhash}', ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Block not found") + + resp = self.test_rest_request(f"/deploymentinfo/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") if __name__ == '__main__': RESTTest().main() diff --git a/test/functional/p2p_sendtxrcncl.py b/test/functional/p2p_sendtxrcncl.py new file mode 100755 index 0000000000..f4c5dd4586 --- /dev/null +++ b/test/functional/p2p_sendtxrcncl.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test SENDTXRCNCL message +""" + +from test_framework.messages import ( + msg_sendtxrcncl, + msg_verack, + msg_version, + msg_wtxidrelay, +) +from test_framework.p2p import ( + P2PInterface, + P2P_SERVICES, + P2P_SUBVERSION, + P2P_VERSION, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class PeerNoVerack(P2PInterface): + def __init__(self, wtxidrelay=True): + super().__init__(wtxidrelay=wtxidrelay) + + def on_version(self, message): + # Avoid sending verack in response to version. + # When calling add_p2p_connection, wait_for_verack=False must be set (see + # comment in add_p2p_connection). + if message.nVersion >= 70016 and self.wtxidrelay: + self.send_message(msg_wtxidrelay()) + +class SendTxrcnclReceiver(P2PInterface): + def __init__(self): + super().__init__() + self.sendtxrcncl_msg_received = None + + def on_sendtxrcncl(self, message): + self.sendtxrcncl_msg_received = message + +class PeerTrackMsgOrder(P2PInterface): + def __init__(self): + super().__init__() + self.messages = [] + + def on_message(self, message): + super().on_message(message) + self.messages.append(message) + +def create_sendtxrcncl_msg(initiator=True): + sendtxrcncl_msg = msg_sendtxrcncl() + sendtxrcncl_msg.initiator = initiator + sendtxrcncl_msg.responder = not initiator + sendtxrcncl_msg.version = 1 + sendtxrcncl_msg.salt = 2 + return sendtxrcncl_msg + +class SendTxRcnclTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [['-txreconciliation']] + + def run_test(self): + self.log.info('SENDTXRCNCL sent to an inbound') + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=True, wait_for_verack=True) + assert peer.sendtxrcncl_msg_received + assert not peer.sendtxrcncl_msg_received.initiator + assert peer.sendtxrcncl_msg_received.responder + assert_equal(peer.sendtxrcncl_msg_received.version, 1) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL should be sent before VERACK') + peer = self.nodes[0].add_p2p_connection(PeerTrackMsgOrder(), send_version=True, wait_for_verack=True) + peer.wait_for_verack() + verack_index = [i for i, msg in enumerate(peer.messages) if msg.msgtype == b'verack'][0] + sendtxrcncl_index = [i for i, msg in enumerate(peer.messages) if msg.msgtype == b'sendtxrcncl'][0] + assert(sendtxrcncl_index < verack_index) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL on pre-WTXID version should not be sent') + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=False, wait_for_verack=False) + pre_wtxid_version_msg = msg_version() + pre_wtxid_version_msg.nVersion = 70015 + pre_wtxid_version_msg.strSubVer = P2P_SUBVERSION + pre_wtxid_version_msg.nServices = P2P_SERVICES + pre_wtxid_version_msg.relay = 1 + peer.send_message(pre_wtxid_version_msg) + peer.wait_for_verack() + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL for fRelay=false should not be sent') + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=False, wait_for_verack=False) + no_txrelay_version_msg = msg_version() + no_txrelay_version_msg.nVersion = P2P_VERSION + no_txrelay_version_msg.strSubVer = P2P_SUBVERSION + no_txrelay_version_msg.nServices = P2P_SERVICES + no_txrelay_version_msg.relay = 0 + peer.send_message(no_txrelay_version_msg) + peer.wait_for_verack() + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('valid SENDTXRCNCL received') + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(create_sendtxrcncl_msg()) + self.wait_until(lambda : "sendtxrcncl" in self.nodes[0].getpeerinfo()[-1]["bytesrecv_per_msg"]) + self.log.info('second SENDTXRCNCL triggers a disconnect') + peer.send_message(create_sendtxrcncl_msg()) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with initiator=responder=0 triggers a disconnect') + sendtxrcncl_no_role = create_sendtxrcncl_msg() + sendtxrcncl_no_role.initiator = False + sendtxrcncl_no_role.responder = False + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(sendtxrcncl_no_role) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with initiator=0 and responder=1 from inbound triggers a disconnect') + sendtxrcncl_wrong_role = create_sendtxrcncl_msg(initiator=False) + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(sendtxrcncl_wrong_role) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with version=0 triggers a disconnect') + sendtxrcncl_low_version = create_sendtxrcncl_msg() + sendtxrcncl_low_version.version = 0 + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(sendtxrcncl_low_version) + peer.wait_for_disconnect() + + self.log.info('sending SENDTXRCNCL after sending VERACK triggers a disconnect') + # We use PeerNoVerack even though verack is sent right after, to make sure it was actually + # sent before sendtxrcncl is sent. + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_and_ping(msg_verack()) + peer.send_message(create_sendtxrcncl_msg()) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL without WTXIDRELAY is ignored (recon state is erased after VERACK)') + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(wtxidrelay=False), send_version=True, wait_for_verack=False) + with self.nodes[0].assert_debug_log(['Forget txreconciliation state of peer']): + peer.send_message(create_sendtxrcncl_msg()) + peer.send_message(msg_verack()) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL sent to an outbound') + peer = self.nodes[0].add_outbound_p2p_connection( + SendTxrcnclReceiver(), wait_for_verack=True, p2p_idx=1, connection_type="outbound-full-relay") + assert peer.sendtxrcncl_msg_received + assert peer.sendtxrcncl_msg_received.initiator + assert not peer.sendtxrcncl_msg_received.responder + assert_equal(peer.sendtxrcncl_msg_received.version, 1) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL should not be sent if block-relay-only') + peer = self.nodes[0].add_outbound_p2p_connection( + SendTxrcnclReceiver(), wait_for_verack=True, p2p_idx=2, connection_type="block-relay-only") + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL if block-relay-only triggers a disconnect') + peer = self.nodes[0].add_outbound_p2p_connection( + PeerNoVerack(), wait_for_verack=False, p2p_idx=3, connection_type="block-relay-only") + peer.send_message(create_sendtxrcncl_msg(initiator=False)) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with initiator=1 and responder=0 from outbound triggers a disconnect') + sendtxrcncl_wrong_role = create_sendtxrcncl_msg(initiator=True) + peer = self.nodes[0].add_outbound_p2p_connection( + P2PInterface(), wait_for_verack=False, p2p_idx=4, connection_type="outbound-full-relay") + peer.send_message(sendtxrcncl_wrong_role) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL not sent if -txreconciliation flag is not set') + self.restart_node(0, []) + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=True, wait_for_verack=True) + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL not sent if blocksonly is set') + self.restart_node(0, ["-txreconciliation", "-blocksonly"]) + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=True, wait_for_verack=True) + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + +if __name__ == '__main__': + SendTxRcnclTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 4583ca25cf..1fe3b21542 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -27,6 +27,7 @@ from test_framework.psbt import ( PSBT_IN_SHA256, PSBT_IN_HASH160, PSBT_IN_HASH256, + PSBT_OUT_TAP_TREE, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -779,9 +780,18 @@ class PSBTTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) self.nodes[0].importdescriptors([{"desc": descsum_create("tr({})".format(privkey)), "timestamp":"now"}]) - psbt = watchonly.sendall([wallet.getnewaddress()])["psbt"] + psbt = watchonly.sendall([wallet.getnewaddress(), addr])["psbt"] psbt = self.nodes[0].walletprocesspsbt(psbt)["psbt"] - self.nodes[0].sendrawtransaction(self.nodes[0].finalizepsbt(psbt)["hex"]) + txid = self.nodes[0].sendrawtransaction(self.nodes[0].finalizepsbt(psbt)["hex"]) + vout = find_vout_for_address(self.nodes[0], txid, addr) + + # Make sure tap tree is in psbt + parsed_psbt = PSBT.from_base64(psbt) + assert_greater_than(len(parsed_psbt.o[vout].map[PSBT_OUT_TAP_TREE]), 0) + assert "taproot_tree" in self.nodes[0].decodepsbt(psbt)["outputs"][vout] + parsed_psbt.make_blank() + comb_psbt = self.nodes[0].combinepsbt([psbt, parsed_psbt.to_base64()]) + assert_equal(comb_psbt, psbt) self.log.info("Test that walletprocesspsbt both updates and signs a non-updated psbt containing Taproot inputs") addr = self.nodes[0].getnewaddress("", "bech32m") @@ -793,6 +803,14 @@ class PSBTTest(BitcoinTestFramework): self.nodes[0].sendrawtransaction(rawtx) self.generate(self.nodes[0], 1) + # Make sure tap tree is not in psbt + parsed_psbt = PSBT.from_base64(psbt) + assert PSBT_OUT_TAP_TREE not in parsed_psbt.o[0].map + assert "taproot_tree" not in self.nodes[0].decodepsbt(psbt)["outputs"][0] + parsed_psbt.make_blank() + comb_psbt = self.nodes[0].combinepsbt([psbt, parsed_psbt.to_base64()]) + assert_equal(comb_psbt, psbt) + self.log.info("Test decoding PSBT with per-input preimage types") # note that the decodepsbt RPC doesn't check whether preimages and hashes match hash_ripemd160, preimage_ripemd160 = random_bytes(20), random_bytes(50) diff --git a/test/functional/rpc_scanblocks.py b/test/functional/rpc_scanblocks.py new file mode 100755 index 0000000000..39f091fd1a --- /dev/null +++ b/test/functional/rpc_scanblocks.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the scanblocks RPC call.""" +from test_framework.messages import COIN +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) + + +class ScanblocksTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [["-blockfilterindex=1"], []] + + def run_test(self): + node = self.nodes[0] + wallet = MiniWallet(node) + wallet.rescan_utxos() + + # send 1.0, mempool only + _, spk_1, addr_1 = getnewdestination() + wallet.send_to(from_node=node, scriptPubKey=spk_1, amount=1 * COIN) + + parent_key = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" + # send 1.0, mempool only + # childkey 5 of `parent_key` + wallet.send_to(from_node=node, + scriptPubKey=bytes.fromhex(node.validateaddress("mkS4HXoTYWRTescLGaUTGbtTTYX5EjJyEE")['scriptPubKey']), + amount=1 * COIN) + + # mine a block and assure that the mined blockhash is in the filterresult + blockhash = self.generate(node, 1)[0] + height = node.getblockheader(blockhash)['height'] + self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) + + out = node.scanblocks("start", [f"addr({addr_1})"]) + assert(blockhash in out['relevant_blocks']) + assert_equal(height, out['to_height']) + assert_equal(0, out['from_height']) + + # mine another block + blockhash_new = self.generate(node, 1)[0] + height_new = node.getblockheader(blockhash_new)['height'] + + # make sure the blockhash is not in the filter result if we set the start_height + # to the just mined block (unlikely to hit a false positive) + assert(blockhash not in node.scanblocks( + "start", [f"addr({addr_1})"], height_new)['relevant_blocks']) + + # make sure the blockhash is present when using the first mined block as start_height + assert(blockhash in node.scanblocks( + "start", [f"addr({addr_1})"], height)['relevant_blocks']) + + # also test the stop height + assert(blockhash in node.scanblocks( + "start", [f"addr({addr_1})"], height, height)['relevant_blocks']) + + # use the stop_height to exclude the relevant block + assert(blockhash not in node.scanblocks( + "start", [f"addr({addr_1})"], 0, height - 1)['relevant_blocks']) + + # make sure the blockhash is present when using the first mined block as start_height + assert(blockhash in node.scanblocks( + "start", [{"desc": f"pkh({parent_key}/*)", "range": [0, 100]}], height)['relevant_blocks']) + + # test node with disabled blockfilterindex + assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", + self.nodes[1].scanblocks, "start", [f"addr({addr_1})"]) + + # test unknown filtertype + assert_raises_rpc_error(-5, "Unknown filtertype", + node.scanblocks, "start", [f"addr({addr_1})"], 0, 10, "extended") + + # test invalid start_height + assert_raises_rpc_error(-1, "Invalid start_height", + node.scanblocks, "start", [f"addr({addr_1})"], 100000000) + + # test invalid stop_height + assert_raises_rpc_error(-1, "Invalid stop_height", + node.scanblocks, "start", [f"addr({addr_1})"], 10, 0) + assert_raises_rpc_error(-1, "Invalid stop_height", + node.scanblocks, "start", [f"addr({addr_1})"], 10, 100000000) + + # test accessing the status (must be empty) + assert_equal(node.scanblocks("status"), None) + + # test aborting the current scan (there is no, must return false) + assert_equal(node.scanblocks("abort"), False) + + # test invalid command + assert_raises_rpc_error(-8, "Invalid command", node.scanblocks, "foobar") + + +if __name__ == '__main__': + ScanblocksTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index acb6d3ea4a..6eb5b493b9 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -33,6 +33,9 @@ class ScantxoutsetTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.wallet.rescan_utxos() + self.log.info("Test if we find coinbase outputs.") + assert_equal(sum(u["coinbase"] for u in self.nodes[0].scantxoutset("start", [self.wallet.get_descriptor()])["unspents"]), 49) + self.log.info("Create UTXOs...") pubk1, spk_P2SH_SEGWIT, addr_P2SH_SEGWIT = getnewdestination("p2sh-segwit") pubk2, spk_LEGACY, addr_LEGACY = getnewdestination("legacy") diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8a928a1e50..252b49cc6d 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1838,3 +1838,31 @@ class msg_cfcheckpt: def __repr__(self): return "msg_cfcheckpt(filter_type={:#x}, stop_hash={:x})".format( self.filter_type, self.stop_hash) + +class msg_sendtxrcncl: + __slots__ = ("initiator", "responder", "version", "salt") + msgtype = b"sendtxrcncl" + + def __init__(self): + self.initiator = False + self.responder = False + self.version = 0 + self.salt = 0 + + def deserialize(self, f): + self.initiator = struct.unpack("<?", f.read(1))[0] + self.responder = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<I", f.read(4))[0] + self.salt = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.initiator) + r += struct.pack("<?", self.responder) + r += struct.pack("<I", self.version) + r += struct.pack("<Q", self.salt) + return r + + def __repr__(self): + return "msg_sendtxrcncl(initiator=%i, responder=%i, version=%lu, salt=%lu)" %\ + (self.initiator, self.responder, self.version, self.salt) diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index fc72a9ab73..8830b0de63 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -62,6 +62,7 @@ from test_framework.messages import ( msg_sendaddrv2, msg_sendcmpct, msg_sendheaders, + msg_sendtxrcncl, msg_tx, MSG_TX, MSG_TYPE_MASK, @@ -126,6 +127,7 @@ MESSAGEMAP = { b"sendaddrv2": msg_sendaddrv2, b"sendcmpct": msg_sendcmpct, b"sendheaders": msg_sendheaders, + b"sendtxrcncl": msg_sendtxrcncl, b"tx": msg_tx, b"verack": msg_verack, b"version": msg_version, @@ -421,6 +423,7 @@ class P2PInterface(P2PConnection): def on_sendaddrv2(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass + def on_sendtxrcncl(self, message): pass def on_tx(self, message): pass def on_wtxidrelay(self, message): pass diff --git a/test/functional/test_framework/psbt.py b/test/functional/test_framework/psbt.py index 68945e7e84..3a5b4ec74d 100644 --- a/test/functional/test_framework/psbt.py +++ b/test/functional/test_framework/psbt.py @@ -123,6 +123,15 @@ class PSBT: psbt = [x.serialize() for x in [self.g] + self.i + self.o] return b"psbt\xff" + b"".join(psbt) + def make_blank(self): + """ + Remove all fields except for PSBT_GLOBAL_UNSIGNED_TX + """ + for m in self.i + self.o: + m.map.clear() + + self.g = PSBTMap(map={0: self.g.map[0]}) + def to_base64(self): return base64.b64encode(self.serialize()).decode("utf8") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e35cae006f..2367a9a8fa 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -618,7 +618,7 @@ class TestNode(): return p2p_conn - def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs): + def add_outbound_p2p_connection(self, p2p_conn, *, wait_for_verack=True, p2p_idx, connection_type="outbound-full-relay", **kwargs): """Add an outbound p2p connection from node. Must be an "outbound-full-relay", "block-relay-only", "addr-fetch" or "feeler" connection. @@ -640,8 +640,9 @@ class TestNode(): p2p_conn.wait_for_connect() self.p2ps.append(p2p_conn) - p2p_conn.wait_for_verack() - p2p_conn.sync_with_ping() + if wait_for_verack: + p2p_conn.wait_for_verack() + p2p_conn.sync_with_ping() return p2p_conn diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index d78c1c634f..e20de8ea8e 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -277,6 +277,7 @@ BASE_SCRIPTS = [ 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', + 'feature_discover.py', 'wallet_resendwallettransactions.py --legacy-wallet', 'wallet_resendwallettransactions.py --descriptors', 'wallet_fallbackfee.py --legacy-wallet', @@ -316,6 +317,8 @@ BASE_SCRIPTS = [ 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', + 'rpc_scanblocks.py', + 'p2p_sendtxrcncl.py', 'rpc_scantxoutset.py', 'feature_txindex_compatibility.py', 'feature_unsupported_utxo_db.py', diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index a0f17ac119..abf38ca79b 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -14,6 +14,7 @@ import sys EXPECTED_CIRCULAR_DEPENDENCIES = ( "chainparamsbase -> util/system -> chainparamsbase", "node/blockstorage -> validation -> node/blockstorage", + "node/utxo_snapshot -> validation -> node/utxo_snapshot", "policy/fees -> txmempool -> policy/fees", "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", |