diff options
46 files changed, 773 insertions, 367 deletions
diff --git a/Makefile.am b/Makefile.am index 3ac698b96f..79c294fd15 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Pattern rule to print variables, e.g. make print-top_srcdir -print-%: +print-%: FORCE @echo '$*'='$($*)' ACLOCAL_AMFLAGS = -I build-aux/m4 diff --git a/configure.ac b/configure.ac index 6f78e8dec4..fa1b580cf5 100644 --- a/configure.ac +++ b/configure.ac @@ -426,6 +426,7 @@ if test "x$enable_werror" = "xyes"; then [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Werror=implicit-fallthrough], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=implicit-fallthrough"], [], [$CXXFLAG_WERROR]) if test x$suppress_external_warnings != xno ; then AX_CHECK_COMPILE_FLAG([-Werror=documentation],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=documentation"],,[[$CXXFLAG_WERROR]]) @@ -456,6 +457,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) if test x$suppress_external_warnings != xno ; then AX_CHECK_COMPILE_FLAG([-Wdocumentation],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"],,[[$CXXFLAG_WERROR]]) @@ -467,7 +469,6 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) if test x$suppress_external_warnings != xyes ; then AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 0b96949a6b..bc3391e089 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -254,6 +254,10 @@ esac # CXXFLAGS HOST_CXXFLAGS="$HOST_CFLAGS" +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; +esac + # LDFLAGS case "$HOST" in *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; diff --git a/depends/Makefile b/depends/Makefile index ac12e91e49..a3b9cd2099 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -1,7 +1,7 @@ .NOTPARALLEL : # Pattern rule to print variables, e.g. make print-top_srcdir -print-%: +print-%: FORCE @echo '$*'='$($*)' # When invoking a sub-make, keep only the command line variable definitions @@ -284,3 +284,4 @@ download: download-osx download-linux download-win $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources +.PHONY: FORCE diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 165c27f5e3..0c3ef8c82f 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -6,9 +6,11 @@ $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=1c1b4e33137ca77881074c140d54c3c9747e845a31338cfe8680f171f0bc3a39 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch +$(package)_linguist_tools = lrelease lupdate lconvert +$(package)_patches = qt.pro qttools_src.pro +$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch $(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches+= drop_lrelease_dependency.patch no_sdk_version_check.patch +$(package)_patches+= no_sdk_version_check.patch $(package)_patches+= fix_lib_paths.patch fix_android_pch.patch $(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch @@ -64,6 +66,7 @@ $(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests +$(package)_config_opts += -nomake tools $(package)_config_opts += -opensource $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) @@ -113,14 +116,13 @@ $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl $(package)_config_opts_darwin += -pch $(package)_config_opts_darwin += -no-feature-corewlan -$(package)_config_opts_darwin += -device-option QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) +$(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSION) $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" -$(package)_config_opts_darwin += -device-option MAC_MIN_VERSION=$(OSX_MIN_VERSION) $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif @@ -201,26 +203,24 @@ endef # # 1. Apply our patches to the extracted source. See each patch for more info. # -# 2. Point to lrelease in qttools/bin/lrelease; otherwise Qt will look for it in -# $(host)/native/bin/lrelease and not find it. +# 2. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. # -# 3. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. -# -# 4. After making a copy of the mkspec for the linux-arm-gnueabi host, named +# 3. After making a copy of the mkspec for the linux-arm-gnueabi host, named # bitcoin-linux-g++, replace instances of linux-arm-gnueabi with $(host). This # way we can generically support hosts like riscv64-linux-gnu, which Qt doesn't # ship a mkspec for. See it's usage in config_opts_* above. # -# 5. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. +# 4. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. # -# 6. Do similar for the win32-g++ mkspec. +# 5. Do similar for the win32-g++ mkspec. # -# 7. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. +# 6. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. # -# 8. Adjust a regex in toolchain.prf, to accommodate Guix's usage of +# 7. Adjust a regex in toolchain.prf, to accommodate Guix's usage of # CROSS_LIBRARY_PATH. See #15277. define $(package)_preprocess_cmds - patch -p1 -i $($(package)_patch_dir)/drop_lrelease_dependency.patch && \ + cp $($(package)_patch_dir)/qt.pro qt.pro && \ + cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ @@ -232,7 +232,6 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ - sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ @@ -249,35 +248,22 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ + export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ cd qtbase && \ - ./configure $($(package)_config_opts) && \ - cd .. && \ - $(MAKE) -C qtbase sub-src-clean && \ - qtbase/bin/qmake -o qttranslations/Makefile qttranslations/qttranslations.pro && \ - qtbase/bin/qmake -o qttranslations/translations/Makefile qttranslations/translations/translations.pro && \ - qtbase/bin/qmake -o qttools/src/linguist/lrelease/Makefile qttools/src/linguist/lrelease/lrelease.pro && \ - qtbase/bin/qmake -o qttools/src/linguist/lupdate/Makefile qttools/src/linguist/lupdate/lupdate.pro && \ - qtbase/bin/qmake -o qttools/src/linguist/lconvert/Makefile qttools/src/linguist/lconvert/lconvert.pro + ./configure -top-level $($(package)_config_opts) endef define $(package)_build_cmds - $(MAKE) -C qtbase/src $(addprefix sub-,$($(package)_qt_libs)) && \ - $(MAKE) -C qttools/src/linguist/lrelease && \ - $(MAKE) -C qttools/src/linguist/lupdate && \ - $(MAKE) -C qttools/src/linguist/lconvert && \ - $(MAKE) -C qttranslations + $(MAKE) endef define $(package)_stage_cmds $(MAKE) -C qtbase/src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && \ - $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttools/src/linguist/lupdate INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttools/src/linguist/lconvert INSTALL_ROOT=$($(package)_staging_dir) install_target && \ + $(MAKE) -C qttools/src/linguist INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_linguist_tools))) && \ $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets endef define $(package)_postprocess_cmds rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \ - rm -f lib/lib*.la lib/*.prl plugins/*/*.prl + rm -f lib/lib*.la endef diff --git a/depends/patches/qt/drop_lrelease_dependency.patch b/depends/patches/qt/drop_lrelease_dependency.patch deleted file mode 100644 index 9b918af77c..0000000000 --- a/depends/patches/qt/drop_lrelease_dependency.patch +++ /dev/null @@ -1,20 +0,0 @@ -commit 67b3ed7406e1d0762188dbad2c44a06824ba0778 -Author: fanquake <fanquake@gmail.com> -Date: Tue Aug 18 15:24:01 2020 +0800 - - Drop dependency on lrelease - - Qts buildsystem insists on using the installed lrelease, but gets - confused about how to find it. Since we manually control the build - order, just drop the dependency. - - See #9469 - -diff --git a/qttranslations/translations/translations.pro b/qttranslations/translations/translations.pro -index 694544c..eff339d 100644 ---- a/qttranslations/translations/translations.pro -+++ b/qttranslations/translations/translations.pro -@@ -107,3 +107,2 @@ updateqm.commands = $$LRELEASE ${QMAKE_FILE_IN} -qm ${QMAKE_FILE_OUT} - silent:updateqm.commands = @echo lrelease ${QMAKE_FILE_IN} && $$updateqm.commands --updateqm.depends = $$LRELEASE_EXE - updateqm.name = LRELEASE ${QMAKE_FILE_IN} diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 0142667547..190ab7a160 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -8,7 +8,6 @@ include(../common/clang-mac.conf) QMAKE_MAC_SDK_PATH=$${MAC_SDK_PATH} QMAKE_XCODE_VERSION = $${XCODE_VERSION} QMAKE_XCODE_DEVELOPER_PATH=/Developer -QMAKE_MACOSX_DEPLOYMENT_TARGET = $${MAC_MIN_VERSION} QMAKE_MAC_SDK=macosx QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx diff --git a/depends/patches/qt/qt.pro b/depends/patches/qt/qt.pro new file mode 100644 index 0000000000..8f2e900a84 --- /dev/null +++ b/depends/patches/qt/qt.pro @@ -0,0 +1,16 @@ +# Create the super cache so modules will add themselves to it. +cache(, super) + +!QTDIR_build: cache(CONFIG, add, $$list(QTDIR_build)) + +prl = no_install_prl +CONFIG += $$prl +cache(CONFIG, add stash, prl) + +TEMPLATE = subdirs +SUBDIRS = qtbase qttools qttranslations + +qttools.depends = qtbase +qttranslations.depends = qttools + +load(qt_configure) diff --git a/depends/patches/qt/qttools_src.pro b/depends/patches/qt/qttools_src.pro new file mode 100644 index 0000000000..6ef71a0942 --- /dev/null +++ b/depends/patches/qt/qttools_src.pro @@ -0,0 +1,6 @@ +TEMPLATE = subdirs +SUBDIRS = linguist + +fb = force_bootstrap +CONFIG += $$fb +cache(CONFIG, add, fb) diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 9a312bc33c..8c9035c45b 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -4,6 +4,8 @@ The configuration file is used by `bitcoind`, `bitcoin-qt` and `bitcoin-cli`. All command-line options (except for `-?`, `-help`, `-version` and `-conf`) may be specified in a configuration file, and all configuration file options (except for `includeconf`) may also be specified on the command line. Command-line options override values set in the configuration file and configuration file options override values set in the GUI. +Changes to the configuration file while `bitcoind` or `bitcoin-qt` is running only take effect after restarting. + ## Configuration File Format The configuration file is a plain text file and consists of `option=value` entries, one per line. Leading and trailing whitespaces are removed. diff --git a/doc/release-process.md b/doc/release-process.md index 546169df8d..75a574ee31 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -37,6 +37,12 @@ Release Process - This update should be reviewed with a reindex-chainstate with assumevalid=0 to catch any defect that causes rejection of blocks in the past history. - Clear the release notes and move them to the wiki (see "Write the release notes" below). +- Translations on Transifex + - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-<RRR>x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. + - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. + - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. + - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/bitcoin/announcements/) as a template. + - Change the auto-update URL for the resource to `master`, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/master/src/qt/locale/bitcoin_en.xlf`. (Do this only after the previous steps, to prevent an auto-update from interfering.) #### After branch-off (on master) @@ -46,6 +52,8 @@ Release Process - Update the versions. - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. +- Translations on Transifex + - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/<branch>/src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release diff --git a/doc/translation_process.md b/doc/translation_process.md index f132693264..97a8fbfff2 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -63,17 +63,12 @@ username = USERNAME The Transifex Bitcoin project config file is included as part of the repo. It can be found at `.tx/config`, however you shouldn’t need to change anything. ### Synchronising translations -To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). -1. `python3 ../bitcoin-maintainer-tools/update-translations.py` -2. `git add` new translations from `src/qt/locale/` -3. Update `src/qt/bitcoin_locale.qrc` manually or via -```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ <file alias="\2">locale\/\1.qm<\/file>/' +To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). To use it and commit the result, simply do: + ``` -4. Update `src/Makefile.qt_locale.include` manually or via -```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ qt\/locale\/\1.ts \\/' +python3 ../bitcoin-maintainer-tools/update-translations.py +git commit -a ``` **Do not directly download translations** one by one from the Transifex website, as we do a few post-processing steps before committing the translations. diff --git a/src/Makefile.am b/src/Makefile.am index 7de5fb36ed..407fdf5a8f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Pattern rule to print variables, e.g. make print-top_srcdir -print-%: +print-%: FORCE @echo '$*'='$($*)' DIST_SUBDIRS = secp256k1 univalue diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 56b8ca8ce6..2a8e4a0aac 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -35,6 +35,7 @@ bench_bench_bitcoin_SOURCES = \ bench/mempool_stress.cpp \ bench/nanobench.h \ bench/nanobench.cpp \ + bench/peer_eviction.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 8a28f4f249..ce1f93f11f 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -22,6 +22,7 @@ LEVELDB_CPPFLAGS_INT += -DHAVE_SNAPPY=0 -DHAVE_CRC32C=1 LEVELDB_CPPFLAGS_INT += -DHAVE_FDATASYNC=@HAVE_FDATASYNC@ LEVELDB_CPPFLAGS_INT += -DHAVE_FULLFSYNC=@HAVE_FULLFSYNC@ LEVELDB_CPPFLAGS_INT += -DHAVE_O_CLOEXEC=@HAVE_O_CLOEXEC@ +LEVELDB_CPPFLAGS_INT += -DFALLTHROUGH_INTENDED=[[fallthrough]] if WORDS_BIGENDIAN LEVELDB_CPPFLAGS_INT += -DLEVELDB_IS_BIG_ENDIAN=1 diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp new file mode 100644 index 0000000000..46fd9d999e --- /dev/null +++ b/src/bench/peer_eviction.cpp @@ -0,0 +1,157 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <net.h> +#include <netaddress.h> +#include <random.h> +#include <test/util/net.h> +#include <test/util/setup_common.h> + +#include <algorithm> +#include <functional> +#include <vector> + +static void EvictionProtectionCommon( + benchmark::Bench& bench, + int num_candidates, + std::function<void(NodeEvictionCandidate&)> candidate_setup_fn) +{ + using Candidates = std::vector<NodeEvictionCandidate>; + FastRandomContext random_context{true}; + bench.warmup(100).epochIterations(1100); + + Candidates candidates{GetRandomNodeEvictionCandidates(num_candidates, random_context)}; + for (auto& c : candidates) { + candidate_setup_fn(c); + } + + std::vector<Candidates> copies{ + static_cast<size_t>(bench.epochs() * bench.epochIterations()), candidates}; + size_t i{0}; + bench.run([&] { + ProtectEvictionCandidatesByRatio(copies.at(i)); + ++i; + }); +} + +/* Benchmarks */ + +static void EvictionProtection0Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_network = NET_IPV4; + }); +} + +static void EvictionProtection1Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = false; + if (c.id >= 130 && c.id < 240) { // 110 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection2Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = false; + if (c.id >= 90 && c.id < 160) { // 70 Tor + c.m_network = NET_ONION; + } else if (c.id >= 170 && c.id < 250) { // 80 I2P + c.m_network = NET_I2P; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks050Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 50 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id == 28 || c.id == 47); // 2 localhost + if (c.id >= 30 && c.id < 47) { // 17 I2P + c.m_network = NET_I2P; + } else if (c.id >= 24 && c.id < 28) { // 4 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks100Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 100 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id >= 55 && c.id < 60); // 5 localhost + if (c.id >= 70 && c.id < 80) { // 10 I2P + c.m_network = NET_I2P; + } else if (c.id >= 80 && c.id < 96) { // 16 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id >= 140 && c.id < 160); // 20 localhost + if (c.id >= 170 && c.id < 180) { // 10 I2P + c.m_network = NET_I2P; + } else if (c.id >= 190 && c.id < 240) { // 50 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +// Candidate numbers used for the benchmarks: +// - 50 candidates simulates a possible use of -maxconnections +// - 100 candidates approximates an average node with default settings +// - 250 candidates is the number of peers reported by operators of busy nodes + +// No disadvantaged networks, with 250 eviction candidates. +BENCHMARK(EvictionProtection0Networks250Candidates); + +// 1 disadvantaged network (Tor) with 250 eviction candidates. +BENCHMARK(EvictionProtection1Networks250Candidates); + +// 2 disadvantaged networks (I2P, Tor) with 250 eviction candidates. +BENCHMARK(EvictionProtection2Networks250Candidates); + +// 3 disadvantaged networks (I2P/localhost/Tor) with 50/100/250 eviction candidates. +BENCHMARK(EvictionProtection3Networks050Candidates); +BENCHMARK(EvictionProtection3Networks100Candidates); +BENCHMARK(EvictionProtection3Networks250Candidates); diff --git a/src/hash.cpp b/src/hash.cpp index cc46043c2b..3465caa3a9 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -47,8 +47,10 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vData switch (vDataToHash.size() & 3) { case 3: k1 ^= tail[2] << 16; + [[fallthrough]]; case 2: k1 ^= tail[1] << 8; + [[fallthrough]]; case 1: k1 ^= tail[0]; k1 *= c1; diff --git a/src/init.cpp b/src/init.cpp index 9a4475466c..9afd76d62d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1349,7 +1349,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const int64_t load_block_index_start_time = GetTimeMillis(); try { LOCK(cs_main); - chainman.InitializeChainstate(*Assert(node.mempool)); + chainman.InitializeChainstate(Assert(node.mempool.get())); chainman.m_total_coinstip_cache = nCoinCacheUsage; chainman.m_total_coinsdb_cache = nCoinDBCache; diff --git a/src/net.cpp b/src/net.cpp index 0078d71142..70ba875c4b 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -937,14 +937,17 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti size_t num_protected{0}; while (num_protected < max_protect_by_network) { + // Count the number of disadvantaged networks from which we have peers to protect. + auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); + if (num_networks == 0) { + break; + } const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; - const size_t protect_per_network{ - std::max(disadvantaged_to_protect / networks.size(), static_cast<size_t>(1))}; - + const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; // Early exit flag if there are no remaining candidates by disadvantaged network. bool protected_at_least_one{false}; - for (const Net& n : networks) { + for (Net& n : networks) { if (n.count == 0) continue; const size_t before = eviction_candidates.size(); EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), @@ -954,10 +957,12 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti const size_t after = eviction_candidates.size(); if (before > after) { protected_at_least_one = true; - num_protected += before - after; + const size_t delta{before - after}; + num_protected += delta; if (num_protected >= max_protect_by_network) { break; } + n.count -= delta; } } if (!protected_at_least_one) { diff --git a/src/net_permissions.h b/src/net_permissions.h index c00689465e..bc979e3792 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -31,7 +31,8 @@ enum class NetPermissionFlags : uint32_t { NoBan = (1U << 4) | Download, // Can query the mempool Mempool = (1U << 5), - // Can request addrs without hitting a privacy-preserving cache + // Can request addrs without hitting a privacy-preserving cache, and send us + // unlimited amounts of addrs. Addr = (1U << 7), // True if the user did not specifically set fine grained permissions diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 315d2ac5cd..9c4544df21 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -155,6 +155,13 @@ static constexpr uint32_t MAX_GETCFHEADERS_SIZE = 2000; static constexpr size_t MAX_PCT_ADDR_TO_SEND = 23; /** The maximum number of address records permitted in an ADDR message. */ static constexpr size_t MAX_ADDR_TO_SEND{1000}; +/** The maximum rate of address records we're willing to process on average. Can be bypassed using + * the NetPermissionFlags::Addr permission. */ +static constexpr double MAX_ADDR_RATE_PER_SECOND{0.1}; +/** The soft limit of the address processing token bucket (the regular MAX_ADDR_RATE_PER_SECOND + * based increments won't go above this, but the MAX_ADDR_TO_SEND increment following GETADDR + * is exempt from this limit. */ +static constexpr size_t MAX_ADDR_PROCESSING_TOKEN_BUCKET{MAX_ADDR_TO_SEND}; // Internal stuff namespace { @@ -233,6 +240,15 @@ struct Peer { std::atomic_bool m_wants_addrv2{false}; /** Whether this peer has already sent us a getaddr message. */ bool m_getaddr_recvd{false}; + /** Number of addr messages that can be processed from this peer. Start at 1 to + * permit self-announcement. */ + double m_addr_token_bucket{1.0}; + /** When m_addr_token_bucket was last updated */ + std::chrono::microseconds m_addr_token_timestamp{GetTime<std::chrono::microseconds>()}; + /** Total number of addresses that were dropped due to rate limiting. */ + std::atomic<uint64_t> m_addr_rate_limited{0}; + /** Total number of addresses that were processed (excludes rate limited ones). */ + std::atomic<uint64_t> m_addr_processed{0}; /** Set of txids to reconsider once their parent transactions have been accepted **/ std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); @@ -1239,6 +1255,8 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c } stats.m_ping_wait = ping_wait; + stats.m_addr_processed = peer->m_addr_processed.load(); + stats.m_addr_rate_limited = peer->m_addr_rate_limited.load(); return true; } @@ -2583,6 +2601,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Get recent addresses m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make(NetMsgType::GETADDR)); peer->m_getaddr_sent = true; + // When requesting a getaddr, accept an additional MAX_ADDR_TO_SEND addresses in response + // (bypassing the MAX_ADDR_PROCESSING_TOKEN_BUCKET limit). + peer->m_addr_token_bucket += MAX_ADDR_TO_SEND; } if (!pfrom.IsInboundConn()) { @@ -2777,11 +2798,34 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector<CAddress> vAddrOk; int64_t nNow = GetAdjustedTime(); int64_t nSince = nNow - 10 * 60; + + // Update/increment addr rate limiting bucket. + const auto current_time = GetTime<std::chrono::microseconds>(); + if (peer->m_addr_token_bucket < MAX_ADDR_PROCESSING_TOKEN_BUCKET) { + // Don't increment bucket if it's already full + const auto time_diff = std::max(current_time - peer->m_addr_token_timestamp, 0us); + const double increment = CountSecondsDouble(time_diff) * MAX_ADDR_RATE_PER_SECOND; + peer->m_addr_token_bucket = std::min<double>(peer->m_addr_token_bucket + increment, MAX_ADDR_PROCESSING_TOKEN_BUCKET); + } + peer->m_addr_token_timestamp = current_time; + + const bool rate_limited = !pfrom.HasPermission(NetPermissionFlags::Addr); + uint64_t num_proc = 0; + uint64_t num_rate_limit = 0; + Shuffle(vAddr.begin(), vAddr.end(), FastRandomContext()); for (CAddress& addr : vAddr) { if (interruptMsgProc) return; + // Apply rate limiting. + if (rate_limited) { + if (peer->m_addr_token_bucket < 1.0) { + ++num_rate_limit; + continue; + } + peer->m_addr_token_bucket -= 1.0; + } // We only bother storing full nodes, though this may include // things which we would not make an outbound connection to, in // part because we may make feeler connections to them. @@ -2795,6 +2839,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Do not process banned/discouraged addresses beyond remembering we received them continue; } + ++num_proc; bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !peer->m_getaddr_sent && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes @@ -2804,6 +2849,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fReachable) vAddrOk.push_back(addr); } + peer->m_addr_processed += num_proc; + peer->m_addr_rate_limited += num_rate_limit; + LogPrint(BCLog::NET, "Received addr: %u addresses (%u processed, %u rate-limited) from peer=%d%s\n", + vAddr.size(), + num_proc, + num_rate_limit, + pfrom.GetId(), + fLogIPs ? ", peeraddr=" + pfrom.addr.ToString() : ""); + m_addrman.Add(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) peer->m_getaddr_sent = false; if (pfrom.IsAddrFetchConn()) { diff --git a/src/net_processing.h b/src/net_processing.h index d5801aadd3..c537efb5db 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -29,6 +29,8 @@ struct CNodeStateStats { int m_starting_height = -1; std::chrono::microseconds m_ping_wait; std::vector<int> vHeightInFlight; + uint64_t m_addr_processed = 0; + uint64_t m_addr_rate_limited = 0; }; class PeerManager : public CValidationInterface, public NetEventsInterface diff --git a/src/netaddress.h b/src/netaddress.h index d504996347..5e1d9d2a6f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -227,7 +227,7 @@ class CNetAddr */ bool IsRelayable() const { - return IsIPv4() || IsIPv6() || IsTor(); + return IsIPv4() || IsIPv6() || IsTor() || IsI2P(); } /** diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 9c57816f91..56f55363b2 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -288,6 +288,7 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes } if (breakParsing) break; + [[fallthrough]]; } case STATE_ARGUMENT: // In or after argument case STATE_EATING_SPACES_IN_ARG: @@ -401,6 +402,7 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes strResult = lastResult.get_str(); else strResult = lastResult.write(2); + [[fallthrough]]; case STATE_ARGUMENT: case STATE_EATING_SPACES: return true; diff --git a/src/rest.cpp b/src/rest.cpp index d599f381e3..e50ab33e54 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -524,6 +524,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: // convert hex to bin, continue then with bin part std::vector<unsigned char> strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); + [[fallthrough]]; } case RetFormat::BINARY: { diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 2762d78493..692096367c 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -850,7 +850,7 @@ static RPCHelpMan getblocktemplate() case ThresholdState::LOCKED_IN: // Ensure bit is set in block version pblock->nVersion |= g_versionbitscache.Mask(consensusParams, pos); - // FALL THROUGH to get vbavailable set... + [[fallthrough]]; case ThresholdState::STARTED: { const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 3013c76825..083ba31b71 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -242,6 +242,8 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); + obj.pushKV("addr_processed", statestats.m_addr_processed); + obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); } UniValue permissions(UniValue::VARR); for (const auto& permission : NetPermissions::ToStrings(stats.m_permissionFlags)) { diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 65276f641f..7864e690d8 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -612,15 +612,18 @@ bool IsSolvable(const SigningProvider& provider, const CScript& script) bool IsSegWitOutput(const SigningProvider& provider, const CScript& script) { - std::vector<valtype> solutions; - auto whichtype = Solver(script, solutions); - if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; - if (whichtype == TxoutType::SCRIPTHASH) { - auto h160 = uint160(solutions[0]); - CScript subscript; - if (provider.GetCScript(CScriptID{h160}, subscript)) { - whichtype = Solver(subscript, solutions); - if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; + int version; + valtype program; + if (script.IsWitnessProgram(version, program)) return true; + if (script.IsPayToScriptHash()) { + std::vector<valtype> solutions; + auto whichtype = Solver(script, solutions); + if (whichtype == TxoutType::SCRIPTHASH) { + auto h160 = uint160(solutions[0]); + CScript subscript; + if (provider.GetCScript(CScriptID{h160}, subscript)) { + if (subscript.IsWitnessProgram(version, program)) return true; + } } } return false; diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index ece3214ed5..0d87f687d3 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -343,3 +343,158 @@ CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) no Assert(call_size == std::variant_size_v<CTxDestination>); return tx_destination; } + +CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept +{ + // Avoid: + // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' + // + // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() + const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); + assert(MoneyRange(fee)); + const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); + const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; +} + +bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept +{ + for (const CTxIn& tx_in : tx.vin) { + const Coin& coin = inputs.AccessCoin(tx_in.prevout); + if (coin.IsSpent()) { + return true; + } + } + return false; +} + +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); + CNetAddr net_addr; + if (network == Network::NET_IPV4) { + in_addr v4_addr = {}; + v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + net_addr = CNetAddr{v4_addr}; + } else if (network == Network::NET_IPV6) { + if (fuzzed_data_provider.remaining_bytes() >= 16) { + in6_addr v6_addr = {}; + memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); + net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + } + } else if (network == Network::NET_INTERNAL) { + net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); + } else if (network == Network::NET_ONION) { + net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); + } + return net_addr; +} + +FILE* FuzzedFileProvider::open() +{ + SetFuzzedErrNo(m_fuzzed_data_provider); + if (m_fuzzed_data_provider.ConsumeBool()) { + return nullptr; + } + std::string mode; + CallOneOf( + m_fuzzed_data_provider, + [&] { + mode = "r"; + }, + [&] { + mode = "r+"; + }, + [&] { + mode = "w"; + }, + [&] { + mode = "w+"; + }, + [&] { + mode = "a"; + }, + [&] { + mode = "a+"; + }); +#if defined _GNU_SOURCE && !defined __ANDROID__ + const cookie_io_functions_t io_hooks = { + FuzzedFileProvider::read, + FuzzedFileProvider::write, + FuzzedFileProvider::seek, + FuzzedFileProvider::close, + }; + return fopencookie(this, mode.c_str(), io_hooks); +#else + (void)mode; + return nullptr; +#endif +} + +ssize_t FuzzedFileProvider::read(void* cookie, char* buf, size_t size) +{ + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + if (buf == nullptr || size == 0 || fuzzed_file->m_fuzzed_data_provider.ConsumeBool()) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + const std::vector<uint8_t> random_bytes = fuzzed_file->m_fuzzed_data_provider.ConsumeBytes<uint8_t>(size); + if (random_bytes.empty()) { + return 0; + } + std::memcpy(buf, random_bytes.data(), random_bytes.size()); + if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)random_bytes.size())) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + fuzzed_file->m_offset += random_bytes.size(); + return random_bytes.size(); +} + +ssize_t FuzzedFileProvider::write(void* cookie, const char* buf, size_t size) +{ + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + const ssize_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(0, size); + if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)n)) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + fuzzed_file->m_offset += n; + return n; +} + +int FuzzedFileProvider::seek(void* cookie, int64_t* offset, int whence) +{ + assert(whence == SEEK_SET || whence == SEEK_CUR || whence == SEEK_END); + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + int64_t new_offset = 0; + if (whence == SEEK_SET) { + new_offset = *offset; + } else if (whence == SEEK_CUR) { + if (AdditionOverflow(fuzzed_file->m_offset, *offset)) { + return -1; + } + new_offset = fuzzed_file->m_offset + *offset; + } else if (whence == SEEK_END) { + const int64_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 4096); + if (AdditionOverflow(n, *offset)) { + return -1; + } + new_offset = n + *offset; + } + if (new_offset < 0) { + return -1; + } + fuzzed_file->m_offset = new_offset; + *offset = new_offset; + return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); +} + +int FuzzedFileProvider::close(void* cookie) +{ + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 9f09395a9a..bb017b3497 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -164,20 +164,7 @@ template <typename WeakEnumType, size_t size> return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } -[[nodiscard]] inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept -{ - // Avoid: - // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' - // - // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() - const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); - assert(MoneyRange(fee)); - const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); - const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); - return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; -} +[[nodiscard]] CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept; [[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; @@ -215,16 +202,7 @@ template <class T> return std::numeric_limits<T>::max() - i < j; } -[[nodiscard]] inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept -{ - for (const CTxIn& tx_in : tx.vin) { - const Coin& coin = inputs.AccessCoin(tx_in.prevout); - if (coin.IsSpent()) { - return true; - } - } - return false; -} +[[nodiscard]] bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept; /** * Sets errno to a value selected from the given std::array `errnos`. @@ -259,27 +237,7 @@ inline void SetFuzzedErrNo(FuzzedDataProvider& fuzzed_data_provider) noexcept return result; } -inline CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); - CNetAddr net_addr; - if (network == Network::NET_IPV4) { - in_addr v4_addr = {}; - v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - net_addr = CNetAddr{v4_addr}; - } else if (network == Network::NET_IPV6) { - if (fuzzed_data_provider.remaining_bytes() >= 16) { - in6_addr v6_addr = {}; - memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); - net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - } - } else if (network == Network::NET_INTERNAL) { - net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); - } else if (network == Network::NET_ONION) { - net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); - } - return net_addr; -} +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept; inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept { @@ -329,112 +287,15 @@ public: { } - FILE* open() - { - SetFuzzedErrNo(m_fuzzed_data_provider); - if (m_fuzzed_data_provider.ConsumeBool()) { - return nullptr; - } - std::string mode; - CallOneOf( - m_fuzzed_data_provider, - [&] { - mode = "r"; - }, - [&] { - mode = "r+"; - }, - [&] { - mode = "w"; - }, - [&] { - mode = "w+"; - }, - [&] { - mode = "a"; - }, - [&] { - mode = "a+"; - }); -#if defined _GNU_SOURCE && !defined __ANDROID__ - const cookie_io_functions_t io_hooks = { - FuzzedFileProvider::read, - FuzzedFileProvider::write, - FuzzedFileProvider::seek, - FuzzedFileProvider::close, - }; - return fopencookie(this, mode.c_str(), io_hooks); -#else - (void)mode; - return nullptr; -#endif - } + FILE* open(); - static ssize_t read(void* cookie, char* buf, size_t size) - { - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - if (buf == nullptr || size == 0 || fuzzed_file->m_fuzzed_data_provider.ConsumeBool()) { - return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - } - const std::vector<uint8_t> random_bytes = fuzzed_file->m_fuzzed_data_provider.ConsumeBytes<uint8_t>(size); - if (random_bytes.empty()) { - return 0; - } - std::memcpy(buf, random_bytes.data(), random_bytes.size()); - if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)random_bytes.size())) { - return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - } - fuzzed_file->m_offset += random_bytes.size(); - return random_bytes.size(); - } + static ssize_t read(void* cookie, char* buf, size_t size); - static ssize_t write(void* cookie, const char* buf, size_t size) - { - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - const ssize_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(0, size); - if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)n)) { - return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - } - fuzzed_file->m_offset += n; - return n; - } + static ssize_t write(void* cookie, const char* buf, size_t size); - static int seek(void* cookie, int64_t* offset, int whence) - { - assert(whence == SEEK_SET || whence == SEEK_CUR || whence == SEEK_END); - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - int64_t new_offset = 0; - if (whence == SEEK_SET) { - new_offset = *offset; - } else if (whence == SEEK_CUR) { - if (AdditionOverflow(fuzzed_file->m_offset, *offset)) { - return -1; - } - new_offset = fuzzed_file->m_offset + *offset; - } else if (whence == SEEK_END) { - const int64_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 4096); - if (AdditionOverflow(n, *offset)) { - return -1; - } - new_offset = n + *offset; - } - if (new_offset < 0) { - return -1; - } - fuzzed_file->m_offset = new_offset; - *offset = new_offset; - return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); - } + static int seek(void* cookie, int64_t* offset, int whence); - static int close(void* cookie) - { - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); - } + static int close(void* cookie); }; [[nodiscard]] inline FuzzedFileProvider ConsumeFile(FuzzedDataProvider& fuzzed_data_provider) noexcept diff --git a/src/test/net_peer_eviction_tests.cpp b/src/test/net_peer_eviction_tests.cpp index 4bfd487b86..5eb280b498 100644 --- a/src/test/net_peer_eviction_tests.cpp +++ b/src/test/net_peer_eviction_tests.cpp @@ -17,28 +17,6 @@ BOOST_FIXTURE_TEST_SUITE(net_peer_eviction_tests, BasicTestingSetup) -std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(const int n_candidates, FastRandomContext& random_context) -{ - std::vector<NodeEvictionCandidate> candidates; - for (int id = 0; id < n_candidates; ++id) { - candidates.push_back({ - /* id */ id, - /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), - /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, - /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), - /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), - /* fRelevantServices */ random_context.randbool(), - /* fRelayTxes */ random_context.randbool(), - /* fBloomFilter */ random_context.randbool(), - /* nKeyedNetGroup */ random_context.randrange(100), - /* prefer_evict */ random_context.randbool(), - /* m_is_local */ random_context.randbool(), - /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], - }); - } - return candidates; -} - // Create `num_peers` random nodes, apply setup function `candidate_setup_fn`, // call ProtectEvictionCandidatesByRatio() to apply protection logic, and then // return true if all of `protected_peer_ids` and none of `unprotected_peer_ids` diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 847a490e03..28d7967078 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -6,6 +6,9 @@ #include <chainparams.h> #include <net.h> +#include <span.h> + +#include <vector> void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const { @@ -37,3 +40,25 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) con NodeReceiveMsgBytes(node, ser_msg.data, complete); return complete; } + +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) +{ + std::vector<NodeEvictionCandidate> candidates; + for (int id = 0; id < n_candidates; ++id) { + candidates.push_back({ + /* id */ id, + /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), + /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, + /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), + /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), + /* fRelevantServices */ random_context.randbool(), + /* fRelayTxes */ random_context.randbool(), + /* fBloomFilter */ random_context.randbool(), + /* nKeyedNetGroup */ random_context.randrange(100), + /* prefer_evict */ random_context.randbool(), + /* m_is_local */ random_context.randbool(), + /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + }); + } + return candidates; +} diff --git a/src/test/util/net.h b/src/test/util/net.h index 1b49a671bd..939ec322ed 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -141,4 +141,6 @@ private: mutable size_t m_consumed; }; +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context); + #endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 748272bb1d..d9d236be1d 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -180,7 +180,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); - m_node.chainman->InitializeChainstate(*m_node.mempool); + m_node.chainman->InitializeChainstate(m_node.mempool.get()); m_node.chainman->ActiveChainstate().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!m_node.chainman->ActiveChainstate().CanFlushToDisk()); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 92d8cf2e7d..2893b412fb 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) return outp; }; - CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 7c1db9d4b9..0bd378631b 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a legacy (IBD) chainstate. // - CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // const uint256 snapshot_blockhash = GetRandHash(); CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( - mempool, snapshot_blockhash)); + &mempool, snapshot_blockhash)); chainstates.push_back(&c2); BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -147,7 +147,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); + CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index a3b344d2c9..22aafcaa6c 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -20,10 +20,9 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate{mempool, blockman}; + CChainState chainstate{&mempool, blockman}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); - CTxMemPool tx_pool{}; constexpr bool is_64_bit = sizeof(void*) == 8; @@ -57,7 +56,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // Without any coins in the cache, we shouldn't need to flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); // If the initial memory allocations of cacheCoins don't match these common @@ -72,7 +71,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); @@ -93,7 +92,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); } @@ -101,26 +100,26 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 4; ++i) { add_coin(view); print_view_mem_usage(view); - if (chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == + if (chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == CoinsCacheSizeState::CRITICAL) { break; } } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); // Passing non-zero max mempool usage should allow us more headroom. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); for (int i{0}; i < 3; ++i) { add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); } @@ -136,7 +135,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) BOOST_CHECK(usage_percentage >= 0.9); BOOST_CHECK(usage_percentage < 1); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 1 << 10), CoinsCacheSizeState::LARGE); } @@ -144,7 +143,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 1000; ++i) { add_coin(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool), + chainstate.GetCoinsCacheSizeState(), CoinsCacheSizeState::OK); } @@ -152,7 +151,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // preallocated memory that doesn't get reclaimed even after flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); view.SetBestBlock(InsecureRand256()); @@ -160,7 +159,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); } diff --git a/src/tinyformat.h b/src/tinyformat.h index bc893ccda5..bedaa14007 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -797,27 +797,27 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& positionalMode break; case 'X': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'x': case 'p': out.setf(std::ios::hex, std::ios::basefield); intConversion = true; break; case 'E': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'e': out.setf(std::ios::scientific, std::ios::floatfield); out.setf(std::ios::dec, std::ios::basefield); break; case 'F': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'f': out.setf(std::ios::fixed, std::ios::floatfield); break; case 'A': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'a': # ifdef _MSC_VER // Workaround https://developercommunity.visualstudio.com/content/problem/520472/hexfloat-stream-output-does-not-ignore-precision-a.html @@ -829,7 +829,7 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& positionalMode break; case 'G': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'g': out.setf(std::ios::dec, std::ios::basefield); // As in boost::format, let stream decide float format. diff --git a/src/validation.cpp b/src/validation.cpp index c80f015143..26333d7026 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -329,23 +329,14 @@ static bool IsCurrentForFeeEstimation(CChainState& active_chainstate) EXCLUSIVE_ return true; } -/* Make mempool consistent after a reorg, by re-adding or recursively erasing - * disconnected block transactions from the mempool, and also removing any - * other transactions from the mempool that are no longer valid given the new - * tip/height. - * - * Note: we assume that disconnectpool only contains transactions that are NOT - * confirmed in the current chain nor already in the mempool (otherwise, - * in-mempool descendants of such transactions would be removed). - * - * Passing fAddToMempool=false will skip trying to add the transactions back, - * and instead just erase from the mempool as needed. - */ - -static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& mempool, DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, mempool.cs) +void CChainState::MaybeUpdateMempoolForReorg( + DisconnectedBlockTransactions& disconnectpool, + bool fAddToMempool) { + if (!m_mempool) return; + AssertLockHeld(cs_main); - AssertLockHeld(mempool.cs); + AssertLockHeld(m_mempool->cs); std::vector<uint256> vHashUpdate; // disconnectpool's insertion_order index sorts the entries from // oldest to newest, but the oldest entry will be the last tx from the @@ -357,11 +348,13 @@ static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& me while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions if (!fAddToMempool || (*it)->IsCoinBase() || - AcceptToMemoryPool(active_chainstate, mempool, *it, true /* bypass_limits */).m_result_type != MempoolAcceptResult::ResultType::VALID) { + AcceptToMemoryPool( + *this, *m_mempool, *it, true /* bypass_limits */).m_result_type != + MempoolAcceptResult::ResultType::VALID) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). - mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); - } else if (mempool.exists((*it)->GetHash())) { + m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (m_mempool->exists((*it)->GetHash())) { vHashUpdate.push_back((*it)->GetHash()); } ++it; @@ -372,12 +365,16 @@ static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& me // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in // the disconnectpool that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); + m_mempool->UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions - mempool.removeForReorg(active_chainstate, STANDARD_LOCKTIME_VERIFY_FLAGS); + m_mempool->removeForReorg(*this, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions - LimitMempoolSize(mempool, active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + LimitMempoolSize( + *m_mempool, + this->CoinsTip(), + gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, + std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); } /** @@ -1212,7 +1209,7 @@ void CoinsViews::InitCache() m_cacheview = std::make_unique<CCoinsViewCache>(&m_catcherview); } -CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash) +CChainState::CChainState(CTxMemPool* mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), m_params(::Params()), m_blockman(blockman), @@ -2005,20 +2002,18 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return true; } -CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool* tx_pool) +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState() { return this->GetCoinsCacheSizeState( - tx_pool, m_coinstip_cache_size_bytes, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( - const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) { - const int64_t nMempoolUsage = tx_pool ? tx_pool->DynamicMemoryUsage() : 0; + const int64_t nMempoolUsage = m_mempool ? m_mempool->DynamicMemoryUsage() : 0; int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); int64_t nTotalSpace = max_coins_cache_size_bytes + std::max<int64_t>(max_mempool_size_bytes - nMempoolUsage, 0); @@ -2057,7 +2052,7 @@ bool CChainState::FlushStateToDisk( bool fFlushForPrune = false; bool fDoFullFlush = false; - CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&m_mempool); + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { // make sure we don't prune above the blockfilterindexes bestblocks @@ -2208,12 +2203,12 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) res += warn; } -/** Check warning conditions and do some notifications on new chain tip set. */ -static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const CChainParams& chainParams, CChainState& active_chainstate) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +void CChainState::UpdateTip(const CBlockIndex* pindexNew) { // New best block - mempool.AddTransactionsUpdated(1); + if (m_mempool) { + m_mempool->AddTransactionsUpdated(1); + } { LOCK(g_best_block_mutex); @@ -2222,11 +2217,11 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C } bilingual_str warning_messages; - if (!active_chainstate.IsInitialBlockDownload()) { + if (!this->IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); - ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); + ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache[bit]); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2241,14 +2236,14 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), pindexNew), active_chainstate.CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), active_chainstate.CoinsTip().GetCacheSize(), + GuessVerificationProgress(m_params.TxData(), pindexNew), this->CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), this->CoinsTip().GetCacheSize(), !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); } /** Disconnect m_chain's tip. * After calling, the mempool will be in an inconsistent state, with * transactions from disconnected blocks being added to disconnectpool. You - * should make the mempool consistent again by calling UpdateMempoolForReorg. + * should make the mempool consistent again by calling MaybeUpdateMempoolForReorg. * with cs_main held. * * If disconnectpool is nullptr, then no disconnected transactions are added to @@ -2258,7 +2253,7 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); @@ -2284,7 +2279,7 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr return false; } - if (disconnectpool) { + if (disconnectpool && m_mempool) { // Save transactions to re-add to mempool at end of reorg for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { disconnectpool->addTransaction(*it); @@ -2292,14 +2287,14 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { // Drop the earliest entry, and remove its children from the mempool. auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); - m_mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); disconnectpool->removeEntry(it); } } m_chain.SetTip(pindexDelete->pprev); - UpdateTip(m_mempool, pindexDelete->pprev, m_params, *this); + UpdateTip(pindexDelete->pprev); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: GetMainSignals().BlockDisconnected(pblock, pindexDelete); @@ -2361,7 +2356,7 @@ public: bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. @@ -2405,11 +2400,13 @@ bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); // Remove conflicting transactions from the mempool.; - m_mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); - disconnectpool.removeForBlock(blockConnecting.vtx); + if (m_mempool) { + m_mempool->removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); + } // Update m_chain & related variables. m_chain.SetTip(pindexNew); - UpdateTip(m_mempool, pindexNew, m_params, *this); + UpdateTip(pindexNew); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); @@ -2499,7 +2496,7 @@ void CChainState::PruneBlockIndexCandidates() { bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); const CBlockIndex* pindexOldTip = m_chain.Tip(); const CBlockIndex* pindexFork = m_chain.FindFork(pindexMostWork); @@ -2511,7 +2508,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex if (!DisconnectTip(state, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, // just in case. Only remove from the mempool in this case. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, false); + MaybeUpdateMempoolForReorg(disconnectpool, false); // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort @@ -2555,7 +2552,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex // A system error occurred (disk space, database error, ...). // Make the mempool consistent with the current tip, just in case // any observers try to use it before shutdown. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, false); + MaybeUpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2572,9 +2569,9 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex if (fBlocksDisconnected) { // If any blocks were disconnected, disconnectpool may be non empty. Add // any disconnected transactions back to the mempool. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, true); + MaybeUpdateMempoolForReorg(disconnectpool, true); } - m_mempool.check(*this); + if (m_mempool) m_mempool->check(*this); CheckForkWarningConditions(); @@ -2646,7 +2643,8 @@ bool CChainState::ActivateBestChain(BlockValidationState& state, std::shared_ptr { LOCK(cs_main); - LOCK(m_mempool.cs); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + LOCK(MempoolMutex()); CBlockIndex* starting_tip = m_chain.Tip(); bool blocks_connected = false; do { @@ -2796,7 +2794,9 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind LimitValidationInterfaceQueue(); LOCK(cs_main); - LOCK(m_mempool.cs); // Lock for as long as disconnectpool is in scope to make sure UpdateMempoolForReorg is called after DisconnectTip without unlocking in between + // Lock for as long as disconnectpool is in scope to make sure MaybeUpdateMempoolForReorg is + // called after DisconnectTip without unlocking in between + LOCK(MempoolMutex()); if (!m_chain.Contains(pindex)) break; pindex_was_in_chain = true; CBlockIndex *invalid_walk_tip = m_chain.Tip(); @@ -2810,7 +2810,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // transactions back to the mempool if disconnecting was successful, // and we're not doing a very deep invalidation (in which case // keeping the mempool up to date is probably futile anyway). - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + MaybeUpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); if (!ret) return false; assert(invalid_walk_tip->pprev == m_chain.Tip()); @@ -3821,10 +3821,11 @@ bool CChainState::LoadBlockIndexDB() void CChainState::LoadMempool(const ArgsManager& args) { + if (!m_mempool) return; if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(m_mempool, *this); + ::LoadMempool(*m_mempool, *this); } - m_mempool.SetIsLoaded(!ShutdownRequested()); + m_mempool->SetIsLoaded(!ShutdownRequested()); } bool CChainState::LoadChainTip() @@ -4688,7 +4689,8 @@ std::vector<CChainState*> ChainstateManager::GetAll() return out; } -CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const std::optional<uint256>& snapshot_blockhash) +CChainState& ChainstateManager::InitializeChainstate( + CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash) { bool is_snapshot = snapshot_blockhash.has_value(); std::unique_ptr<CChainState>& to_modify = @@ -4767,7 +4769,7 @@ bool ChainstateManager::ActivateSnapshot( } auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique<CChainState>( - this->ActiveChainstate().m_mempool, m_blockman, base_blockhash)); + /* mempool */ nullptr, m_blockman, base_blockhash)); { LOCK(::cs_main); @@ -4883,7 +4885,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } const auto snapshot_cache_state = WITH_LOCK(::cs_main, - return snapshot_chainstate.GetCoinsCacheSizeState(&snapshot_chainstate.m_mempool)); + return snapshot_chainstate.GetCoinsCacheSizeState()); if (snapshot_cache_state >= CoinsCacheSizeState::CRITICAL) { diff --git a/src/validation.h b/src/validation.h index 3d66e3161d..9a2be3ad97 100644 --- a/src/validation.h +++ b/src/validation.h @@ -587,8 +587,9 @@ protected: */ mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! mempool that is kept in sync with the chain - CTxMemPool& m_mempool; + //! Optional mempool that is kept in sync with the chain. + //! Only the active chainstate has a mempool. + CTxMemPool* m_mempool; const CChainParams& m_params; @@ -600,7 +601,10 @@ public: //! CChainState instances. BlockManager& m_blockman; - explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash = std::nullopt); + explicit CChainState( + CTxMemPool* mempool, + BlockManager& blockman, + std::optional<uint256> from_snapshot_blockhash = std::nullopt); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -729,7 +733,7 @@ public: CCoinsViewCache& view, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. - bool DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); // Manual block validity manipulation: /** Mark a block as precious and reorganize. @@ -773,19 +777,17 @@ public: //! Dictates whether we need to flush the cache to disk or not. //! //! @return the state of the size of the coins cache. - CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool* tx_pool) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + CoinsCacheSizeState GetCoinsCacheSizeState() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); CoinsCacheSizeState GetCoinsCacheSizeState( - const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); private: - bool ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); - bool ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); + bool ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); void InvalidBlockFound(CBlockIndex* pindex, const BlockValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -798,6 +800,33 @@ private: bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Indirection necessary to make lock annotations work with an optional mempool. + RecursiveMutex* MempoolMutex() const LOCK_RETURNED(m_mempool->cs) + { + return m_mempool ? &m_mempool->cs : nullptr; + } + + /** + * Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + void MaybeUpdateMempoolForReorg( + DisconnectedBlockTransactions& disconnectpool, + bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); + + /** Check warning conditions and do some notifications on new chain tip set. */ + void UpdateTip(const CBlockIndex* pindexNew) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + friend ChainstateManager; }; @@ -907,7 +936,9 @@ public: // constructor //! @param[in] snapshot_blockhash If given, signify that this chainstate //! is based on a snapshot. - CChainState& InitializeChainstate(CTxMemPool& mempool, const std::optional<uint256>& snapshot_blockhash = std::nullopt) + CChainState& InitializeChainstate( + CTxMemPool* mempool, + const std::optional<uint256>& snapshot_blockhash = std::nullopt) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 572a695662..e329e0cf8f 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -207,7 +207,7 @@ public: virtual bool CanGetAddresses(bool internal = false) const { return false; } /** Upgrades the wallet to the specified version */ - virtual bool Upgrade(int prev_version, int new_version, bilingual_str& error) { return false; } + virtual bool Upgrade(int prev_version, int new_version, bilingual_str& error) { return true; } virtual bool HavePrivateKeys() const { return false; } diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 1a414959b9..ff1d85a9be 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -13,17 +13,20 @@ from test_framework.messages import ( msg_addr, msg_getaddr ) -from test_framework.p2p import P2PInterface -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, +from test_framework.p2p import ( + P2PInterface, + p2p_lock, ) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +import random import time class AddrReceiver(P2PInterface): num_ipv4_received = 0 test_addr_contents = False + _tokens = 1 def __init__(self, test_addr_contents=False): super().__init__() @@ -40,6 +43,20 @@ class AddrReceiver(P2PInterface): raise AssertionError("Invalid addr.port of {} (8333-8342 expected)".format(addr.port)) assert addr.ip.startswith('123.123.123.') + def on_getaddr(self, message): + # When the node sends us a getaddr, it increments the addr relay tokens for the connection by 1000 + self._tokens += 1000 + + @property + def tokens(self): + with p2p_lock: + return self._tokens + + def increment_tokens(self, n): + # When we move mocktime forward, the node increments the addr relay tokens for its peers + with p2p_lock: + self._tokens += n + def addr_received(self): return self.num_ipv4_received != 0 @@ -53,12 +70,14 @@ class AddrTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): self.oversized_addr_test() self.relay_tests() self.getaddr_tests() self.blocksonly_mode_tests() + self.rate_limit_tests() def setup_addr_msg(self, num): addrs = [] @@ -75,6 +94,19 @@ class AddrTest(BitcoinTestFramework): msg.addrs = addrs return msg + def setup_rand_addr_msg(self, num): + addrs = [] + for i in range(num): + addr = CAddress() + addr.time = self.mocktime + i + addr.nServices = NODE_NETWORK | NODE_WITNESS + addr.ip = f"{random.randrange(128,169)}.{random.randrange(1,255)}.{random.randrange(1,255)}.{random.randrange(1,255)}" + addr.port = 8333 + addrs.append(addr) + msg = msg_addr() + msg.addrs = addrs + return msg + def send_addr_msg(self, source, msg, receivers): source.send_and_ping(msg) # pop m_next_addr_send timer @@ -191,7 +223,7 @@ class AddrTest(BitcoinTestFramework): def blocksonly_mode_tests(self): self.log.info('Test addr relay in -blocksonly mode') - self.restart_node(0, ["-blocksonly"]) + self.restart_node(0, ["-blocksonly", "-whitelist=addr@127.0.0.1"]) self.mocktime = int(time.time()) self.log.info('Check that we send getaddr messages') @@ -207,6 +239,63 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def send_addrs_and_test_rate_limiting(self, peer, no_relay, new_addrs, total_addrs): + """Send an addr message and check that the number of addresses processed and rate-limited is as expected""" + + peer.send_and_ping(self.setup_rand_addr_msg(new_addrs)) + + peerinfo = self.nodes[0].getpeerinfo()[0] + addrs_processed = peerinfo['addr_processed'] + addrs_rate_limited = peerinfo['addr_rate_limited'] + self.log.debug(f"addrs_processed = {addrs_processed}, addrs_rate_limited = {addrs_rate_limited}") + + if no_relay: + assert_equal(addrs_processed, 0) + assert_equal(addrs_rate_limited, 0) + else: + assert_equal(addrs_processed, min(total_addrs, peer.tokens)) + assert_equal(addrs_rate_limited, max(0, total_addrs - peer.tokens)) + + def rate_limit_tests(self): + + self.mocktime = int(time.time()) + self.restart_node(0, []) + self.nodes[0].setmocktime(self.mocktime) + + for contype, no_relay in [("outbound-full-relay", False), ("block-relay-only", True), ("inbound", False)]: + self.log.info(f'Test rate limiting of addr processing for {contype} peers') + if contype == "inbound": + peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + else: + peer = self.nodes[0].add_outbound_p2p_connection(AddrReceiver(), p2p_idx=0, connection_type=contype) + + # Send 600 addresses. For all but the block-relay-only peer this should result in addresses being processed. + self.send_addrs_and_test_rate_limiting(peer, no_relay, 600, 600) + + # Send 600 more addresses. For the outbound-full-relay peer (which we send a GETADDR, and thus will + # process up to 1001 incoming addresses), this means more addresses will be processed. + self.send_addrs_and_test_rate_limiting(peer, no_relay, 600, 1200) + + # Send 10 more. As we reached the processing limit for all nodes, no more addresses should be procesesd. + self.send_addrs_and_test_rate_limiting(peer, no_relay, 10, 1210) + + # Advance the time by 100 seconds, permitting the processing of 10 more addresses. + # Send 200 and verify that 10 are processed. + self.mocktime += 100 + self.nodes[0].setmocktime(self.mocktime) + peer.increment_tokens(10) + + self.send_addrs_and_test_rate_limiting(peer, no_relay, 200, 1410) + + # Advance the time by 1000 seconds, permitting the processing of 100 more addresses. + # Send 200 and verify that 100 are processed. + self.mocktime += 1000 + self.nodes[0].setmocktime(self.mocktime) + peer.increment_tokens(100) + + self.send_addrs_and_test_rate_limiting(peer, no_relay, 200, 1610) + + self.nodes[0].disconnect_p2ps() if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index 23ce3e5d04..32c1d42b1c 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -18,12 +18,19 @@ from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +I2P_ADDR = "c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p" + ADDRS = [] for i in range(10): addr = CAddress() addr.time = int(time.time()) + i addr.nServices = NODE_NETWORK | NODE_WITNESS - addr.ip = "123.123.123.{}".format(i % 256) + # Add one I2P address at an arbitrary position. + if i == 5: + addr.net = addr.NET_I2P + addr.ip = I2P_ADDR + else: + addr.ip = f"123.123.123.{i % 256}" addr.port = 8333 + i ADDRS.append(addr) @@ -35,11 +42,10 @@ class AddrReceiver(P2PInterface): super().__init__(support_addrv2 = True) def on_addrv2(self, message): - for addr in message.addrs: - assert_equal(addr.nServices, 9) - assert addr.ip.startswith('123.123.123.') - assert (8333 <= addr.port < 8343) - self.addrv2_received_and_checked = True + expected_set = set((addr.ip, addr.port) for addr in ADDRS) + received_set = set((addr.ip, addr.port) for addr in message.addrs) + if expected_set == received_set: + self.addrv2_received_and_checked = True def wait_for_addrv2(self): self.wait_until(lambda: "addrv2" in self.last_message) @@ -49,6 +55,7 @@ class AddrTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): self.log.info('Create connection that sends addrv2 messages') @@ -64,15 +71,18 @@ class AddrTest(BitcoinTestFramework): addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) msg.addrs = ADDRS with self.nodes[0].assert_debug_log([ - 'Added 10 addresses from 127.0.0.1: 0 tried', - 'received: addrv2 (131 bytes) peer=0', - 'sending addrv2 (131 bytes) peer=1', + # The I2P address is not added to node's own addrman because it has no + # I2P reachability (thus 10 - 1 = 9). + 'Added 9 addresses from 127.0.0.1: 0 tried', + 'received: addrv2 (159 bytes) peer=0', + 'sending addrv2 (159 bytes) peer=1', ]): addr_source.send_and_ping(msg) self.nodes[0].setmocktime(int(time.time()) + 30 * 60) addr_receiver.wait_for_addrv2() assert addr_receiver.addrv2_received_and_checked + assert_equal(len(self.nodes[0].getnodeaddresses(count=0, network="i2p")), 0) if __name__ == '__main__': diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 788a81d4af..9c34506320 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -58,6 +58,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): self.test_buffer() diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 504c8c70d4..065e8961ae 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -18,6 +18,7 @@ ser_*, deser_*: functions that handle serialization/deserialization. Classes use __slots__ to ensure extraneous attributes aren't accidentally added by tests, compromising their intended effect. """ +from base64 import b32decode, b32encode from codecs import encode import copy import hashlib @@ -213,15 +214,20 @@ class CAddress: # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki NET_IPV4 = 1 + NET_I2P = 5 ADDRV2_NET_NAME = { - NET_IPV4: "IPv4" + NET_IPV4: "IPv4", + NET_I2P: "I2P" } ADDRV2_ADDRESS_LENGTH = { - NET_IPV4: 4 + NET_IPV4: 4, + NET_I2P: 32 } + I2P_PAD = "====" + def __init__(self): self.time = 0 self.nServices = 1 @@ -229,6 +235,9 @@ class CAddress: self.ip = "0.0.0.0" self.port = 0 + def __eq__(self, other): + return self.net == other.net and self.ip == other.ip and self.nServices == other.nServices and self.port == other.port and self.time == other.time + def deserialize(self, f, *, with_time=True): """Deserialize from addrv1 format (pre-BIP155)""" if with_time: @@ -261,24 +270,33 @@ class CAddress: self.nServices = deser_compact_size(f) self.net = struct.unpack("B", f.read(1))[0] - assert self.net == self.NET_IPV4 + assert self.net in (self.NET_IPV4, self.NET_I2P) address_length = deser_compact_size(f) assert address_length == self.ADDRV2_ADDRESS_LENGTH[self.net] - self.ip = socket.inet_ntoa(f.read(4)) + addr_bytes = f.read(address_length) + if self.net == self.NET_IPV4: + self.ip = socket.inet_ntoa(addr_bytes) + else: + self.ip = b32encode(addr_bytes)[0:-len(self.I2P_PAD)].decode("ascii").lower() + ".b32.i2p" self.port = struct.unpack(">H", f.read(2))[0] def serialize_v2(self): """Serialize in addrv2 format (BIP155)""" - assert self.net == self.NET_IPV4 + assert self.net in (self.NET_IPV4, self.NET_I2P) r = b"" r += struct.pack("<I", self.time) r += ser_compact_size(self.nServices) r += struct.pack("B", self.net) r += ser_compact_size(self.ADDRV2_ADDRESS_LENGTH[self.net]) - r += socket.inet_aton(self.ip) + if self.net == self.NET_IPV4: + r += socket.inet_aton(self.ip) + else: + sfx = ".b32.i2p" + assert self.ip.endswith(sfx) + r += b32decode(self.ip[0:-len(sfx)] + self.I2P_PAD, True) r += struct.pack(">H", self.port) return r diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index ad11f4b244..4d34670ea9 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -94,10 +94,11 @@ class UpgradeWalletTest(BitcoinTestFramework): def test_upgradewallet(self, wallet, previous_version, requested_version=None, expected_version=None): unchanged = expected_version == previous_version new_version = previous_version if unchanged else expected_version if expected_version else requested_version - assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) + old_wallet_info = wallet.getwalletinfo() + assert_equal(old_wallet_info["walletversion"], previous_version) assert_equal(wallet.upgradewallet(requested_version), { - "wallet_name": "", + "wallet_name": old_wallet_info["walletname"], "previous_version": previous_version, "current_version": new_version, "result": "Already at latest version. Wallet version unchanged." if unchanged else "Wallet upgraded successfully from version {} to version {}.".format(previous_version, new_version), @@ -352,6 +353,11 @@ class UpgradeWalletTest(BitcoinTestFramework): v16_3_kvs = dump_bdb_kv(v16_3_wallet) assert b'\x0adefaultkey' not in v16_3_kvs + if self.is_sqlite_compiled(): + self.log.info("Checking that descriptor wallets do nothing, successfully") + self.nodes[0].createwallet(wallet_name="desc_upgrade", descriptors=True) + desc_wallet = self.nodes[0].get_wallet_rpc("desc_upgrade") + self.test_upgradewallet(desc_wallet, previous_version=169900, expected_version=169900) if __name__ == '__main__': UpgradeWalletTest().main() |