diff options
328 files changed, 2976 insertions, 2300 deletions
diff --git a/.gitignore b/.gitignore index 76fc450404..f84a53178e 100644 --- a/.gitignore +++ b/.gitignore @@ -96,7 +96,6 @@ Makefile !depends/Makefile src/qt/bitcoin-qt Bitcoin-Qt.app -background.tiff* # Qt Creator Makefile.am.user @@ -149,6 +148,5 @@ db4/ osx_volname dist/ -*.background.tiff /guix-build-* diff --git a/Makefile.am b/Makefile.am index 9f50e51ce0..1412624d54 100644 --- a/Makefile.am +++ b/Makefile.am @@ -38,9 +38,7 @@ OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso -OSX_BACKGROUND_SVG=background.svg -OSX_BACKGROUND_IMAGE=background.tiff -OSX_BACKGROUND_IMAGE_DPIS=36 72 +OSX_BACKGROUND_IMAGE=$(top_srcdir)/contrib/macdeploy/background.tiff OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed @@ -66,7 +64,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/doc/README_windows.txt OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ - $(top_srcdir)/contrib/macdeploy/$(OSX_BACKGROUND_SVG) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh @@ -127,20 +124,13 @@ osx_volname: echo $(OSX_VOLNAME) >$@ if BUILD_DARWIN -$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE) +$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -$(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@ -$(OSX_BACKGROUND_IMAGE)@2x.png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 72 -p 72 -o $@ -$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE).png $(OSX_BACKGROUND_IMAGE)@2x.png - tiffutil -cathidpicheck $^ -out $@ - deploydir: $(OSX_DMG) else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist -APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE) $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications +APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.tiff $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications $(APP_DIST_DIR)/Applications: @rm -f $@ @@ -154,12 +144,9 @@ $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) $(OSX_DMG): $(OSX_TEMP_ISO) $(DMG) dmg "$<" "$@" -dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ -OSX_BACKGROUND_IMAGE_DPIFILES := $(foreach dpi,$(OSX_BACKGROUND_IMAGE_DPIS),dpi$(dpi).$(OSX_BACKGROUND_IMAGE)) -$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIFILES) +$(APP_DIST_DIR)/.background/background.tiff: $(MKDIR_P) $(@D) - $(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@ + cp $(OSX_BACKGROUND_IMAGE) $@ $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) @@ -336,6 +323,7 @@ EXTRA_DIST += \ test/util/data/txcreatescript4.json \ test/util/data/txcreatescript5.hex \ test/util/data/txcreatescript6.hex \ + test/util/data/txcreatesignsegwit1.hex \ test/util/data/txcreatesignv1.hex \ test/util/data/txcreatesignv1.json \ test/util/data/txcreatesignv2.hex \ @@ -363,7 +351,7 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ - rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff + rm -rf osx_volname dist/ test-security-check: if TARGET_DARWIN @@ -134,3 +134,8 @@ /src/script/interpreter.* @sipa /src/validation.* @sipa /src/consensus/ @sipa + +# Tracing +/doc/tracing.md @jb55 @0xB10C +/src/util/trace.h @jb55 @0xB10C +/contrib/tracing/ @jb55 @0xB10C diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index ae8120fdf3..f70eea5b04 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -13,7 +13,7 @@ packages/* libbitcoin_cli/libbitcoin_cli.vcxproj libbitcoin_common/libbitcoin_common.vcxproj libbitcoin_crypto/libbitcoin_crypto.vcxproj -libbitcoin_server/libbitcoin_server.vcxproj +libbitcoin_node/libbitcoin_node.vcxproj libbitcoin_util/libbitcoin_util.vcxproj libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj libbitcoin_wallet/libbitcoin_wallet.vcxproj diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in index 128c1bd8e7..fc9d7cbed6 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in @@ -21,7 +21,7 @@ <ProjectReference Include="..\libbitcoin_crypto\libbitcoin_crypto.vcxproj"> <Project>{6190199c-6cf4-4dad-bfbd-93fa72a760c1}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_server\libbitcoin_server.vcxproj"> + <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj"> <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index 2800a42767..0d6358e0d0 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -28,7 +28,7 @@ <ProjectReference Include="..\libbitcoin_qt\libbitcoin_qt.vcxproj"> <Project>{2b4abff8-d1fd-4845-88c9-1f3c0a6512bf}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_server\libbitcoin_server.vcxproj"> + <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj"> <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index b2ab64a34b..2a1ccf58fe 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -12,7 +12,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_common", "libbit EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_crypto", "libbitcoin_crypto\libbitcoin_crypto.vcxproj", "{6190199C-6CF4-4DAD-BFBD-93FA72A760C1}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_server", "libbitcoin_server\libbitcoin_server.vcxproj", "{460FEE33-1FE1-483F-B3BF-931FF8E969A5}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_node", "libbitcoin_node\libbitcoin_node.vcxproj", "{460FEE33-1FE1-483F-B3BF-931FF8E969A5}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libunivalue", "libunivalue\libunivalue.vcxproj", "{5724BA7D-A09A-4BA8-800B-C4C1561B3D69}" EndProject diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index d56c359fe0..b1204d0d5d 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -24,7 +24,7 @@ <ProjectReference Include="..\libbitcoin_crypto\libbitcoin_crypto.vcxproj"> <Project>{6190199c-6cf4-4dad-bfbd-93fa72a760c1}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_server\libbitcoin_server.vcxproj"> + <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj"> <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in index 58e90dbaeb..58e90dbaeb 100644 --- a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in +++ b/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index ed4fa7ec9a..2a70cd9332 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -15,7 +15,7 @@ libs = [ 'libbitcoin_cli', 'libbitcoin_common', 'libbitcoin_crypto', - 'libbitcoin_server', + 'libbitcoin_node', 'libbitcoin_util', 'libbitcoin_wallet_tool', 'libbitcoin_wallet', diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index f9948b6f13..d3be693e99 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -40,7 +40,7 @@ <ProjectReference Include="..\libbitcoin_qt\libbitcoin_qt.vcxproj"> <Project>{2b4abff8-d1fd-4845-88c9-1f3c0a6512bf}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_server\libbitcoin_server.vcxproj"> + <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj"> <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 2fc0078b8d..4182448ec3 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -34,7 +34,7 @@ <ProjectReference Include="..\libbitcoin_crypto\libbitcoin_crypto.vcxproj"> <Project>{6190199c-6cf4-4dad-bfbd-93fa72a760c1}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_server\libbitcoin_server.vcxproj"> + <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj"> <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 26d1a44187..d70b993b99 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos export HOST=x86_64-apple-darwin -export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso" +export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.1 export XCODE_BUILD_ID=12A7403 export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 44b6eb7ae3..6619852423 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -13,4 +13,4 @@ export DPKG_ADD_ARCH="i386" export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --disable-external-signer" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 308d216562..491a587b70 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -87,13 +87,17 @@ fi DOCKER_EXEC echo "Free disk space:" DOCKER_EXEC df -h -if [ "$RUN_FUZZ_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - if [ ! -d "${DIR_QA_ASSETS}" ]; then +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" fi - - export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + DOCKER_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" + DOCKER_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi fi DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" diff --git a/configure.ac b/configure.ac index 9e48099fd3..bef3973996 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) -m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [m4_fatal([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) PKG_PROG_PKG_CONFIG if test "$PKG_CONFIG" = ""; then AC_MSG_ERROR([pkg-config not found]) @@ -137,11 +137,11 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) -AC_ARG_ENABLE([ebpf], - [AS_HELP_STRING([--enable-ebpf], - [enable eBPF tracing (default is yes if sys/sdt.h is found)])], - [use_ebpf=$enableval], - [use_ebpf=yes]) +AC_ARG_ENABLE([usdt], + [AS_HELP_STRING([--enable-usdt], + [enable tracepoints for Userspace, Statically Defined Tracing (default is yes if sys/sdt.h is found)])], + [use_usdt=$enableval], + [use_usdt=yes]) AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], @@ -321,7 +321,7 @@ AC_ARG_ENABLE([werror], AC_ARG_ENABLE([external-signer], [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes, requires Boost::Process)])], [use_external_signer=$enableval], - [use_external_signer=yes]) + [use_external_signer=auto]) AC_ARG_ENABLE([lto], [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])], @@ -684,7 +684,6 @@ case $host in TARGET_OS=darwin if test $cross_compiling != "yes"; then BUILD_OS=darwin - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg], [rsvg-convert]) AC_CHECK_PROG([BREW], [brew], [brew]) if test "$BREW" = "brew"; then dnl These Homebrew packages may be keg-only, meaning that they won't be found @@ -746,9 +745,6 @@ case $host in AC_PATH_TOOL([OTOOL], [otool], [otool]) AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) AC_PATH_PROGS([DMG], [dmg], [dmg]) - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg], [rsvg-convert]) - AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert], [convert]) - AC_PATH_PROGS([TIFFCP], [tiffcp], [tiffcp]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -1341,15 +1337,15 @@ if test "$enable_wallet" != "no"; then fi fi -if test "$use_ebpf" != "no"; then - AC_MSG_CHECKING([whether eBPF tracepoints are supported]) +if test "$use_usdt" != "no"; then + AC_MSG_CHECKING([whether Userspace, Statically Defined Tracing tracepoints are supported]) AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include <sys/sdt.h>], [DTRACE_PROBE("context", "event");] )], - [AC_MSG_RESULT([yes]); have_sdt=yes; AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints])], - [AC_MSG_RESULT([no]); have_sdt=no;] + [AC_MSG_RESULT([yes]); AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable tracepoints for Userspace, Statically Defined Tracing])], + [AC_MSG_RESULT([no]); use_usdt=no;] ) fi @@ -1419,7 +1415,21 @@ if test "$use_boost" = "yes"; then fi if test "$use_external_signer" != "no"; then - AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [], [Define if external signer support is enabled]) + case $host in + *mingw*) + dnl Boost Process uses Boost Filesystem when targeting Windows. Also, + dnl since Boost 1.71.0, Process does not work with mingw-w64 without + dnl workarounds. See 67669ab425b52a2b6be3d2f3b3b7e3939b676a2c. + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported on Windows]) + fi + use_external_signer="no"; + ;; + *) + use_external_signer="yes" + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) + ;; + esac fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) @@ -1504,6 +1514,26 @@ if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench fi fi +if test x$use_libevent = xyes; then + TEMP_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $EVENT_CFLAGS" + AC_MSG_CHECKING([if evhttp_connection_get_peer expects const char**]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include <cstdint> + #include <event2/http.h> + ]], [[ + evhttp_connection *conn = (evhttp_connection *)1; + const char *host; + uint16_t port; + + evhttp_connection_get_peer(conn, &host, &port); + ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR], [1], [Define this symbol if evhttp_connection_get_peer expects const char**]) ], + [ AC_MSG_RESULT([no]) ] + ) + CXXFLAGS="$TEMP_CXXFLAGS" +fi + dnl QR Code encoding library check if test "$use_qr" != "no"; then @@ -1763,7 +1793,6 @@ AM_CONDITIONAL([TARGET_WINDOWS], [test "$TARGET_OS" = "windows"]) AM_CONDITIONAL([ENABLE_WALLET], [test "$enable_wallet" = "yes"]) AM_CONDITIONAL([USE_SQLITE], [test "$use_sqlite" = "yes"]) AM_CONDITIONAL([USE_BDB], [test "$use_bdb" = "yes"]) -AM_CONDITIONAL([ENABLE_TRACING], [test "$have_sdt" = "yes"]) AM_CONDITIONAL([ENABLE_TESTS], [test "$BUILD_TEST" = "yes"]) AM_CONDITIONAL([ENABLE_FUZZ], [test "$enable_fuzz" = "yes"]) AM_CONDITIONAL([ENABLE_FUZZ_BINARY], [test "$enable_fuzz_binary" = "yes"]) @@ -1857,6 +1886,7 @@ AC_SUBST(HAVE_BUILTIN_PREFETCH) AC_SUBST(HAVE_MM_PREFETCH) AC_SUBST(HAVE_STRONG_GETAUXVAL) AC_SUBST(ANDROID_ARCH) +AC_SUBST(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) @@ -1865,6 +1895,7 @@ AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check. AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) +AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) @@ -1932,7 +1963,7 @@ echo " with bench = $use_bench" echo " with upnp = $use_upnp" echo " with natpmp = $use_natpmp" echo " use asm = $use_asm" -echo " ebpf tracing = $have_sdt" +echo " USDT tracing = $use_usdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index daee2bb61b..a5b9eac302 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -94,15 +94,15 @@ class TestSecurityChecks(unittest.TestCase): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), (1, executable+': failed CONTROL_FLOW')) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index a6579d6b8b..3e2542a418 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -239,7 +239,7 @@ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --f time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ + --commit=1ef7a03a148cf5f83ab1820444f6bd50d8e732d1 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index c375dc9716..79a3e77f40 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -109,7 +109,7 @@ case "$HOST" in # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" ;; *darwin*) # The CROSS toolchain for darwin uses the SDK and ignores environment variables. @@ -126,7 +126,7 @@ case "$HOST" in export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" ;; *) exit 1 ;; diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 5805006053..c1950ad08e 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -16,8 +16,6 @@ (gnu packages gawk) (gnu packages gcc) (gnu packages gnome) - (gnu packages image) - (gnu packages imagemagick) (gnu packages installers) (gnu packages linux) (gnu packages llvm) @@ -26,11 +24,11 @@ (gnu packages perl) (gnu packages pkg-config) (gnu packages python) + (gnu packages python-crypto) (gnu packages python-web) (gnu packages shells) (gnu packages tls) (gnu packages version-control) - (guix build-system font) (guix build-system gnu) (guix build-system python) (guix build-system trivial) @@ -80,10 +78,6 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (("-rpath=") "-rpath-link=")) #t)))))))) -(define (make-binutils-with-mingw-w64-disable-flags xbinutils) - (package-with-extra-patches xbinutils - (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) - (define (make-cross-toolchain target base-gcc-for-libc base-kernel-headers @@ -135,9 +129,7 @@ chain for " target " development.")) (home-page (package-home-page xgcc)) (license (package-license xgcc))))) -(define base-gcc - (package-with-extra-patches gcc-8 - (search-our-patches "gcc-8-sort-libtool-find-output.patch"))) +(define base-gcc gcc-10) ;; Building glibc with stack smashing protector first landed in glibc 2.25, use ;; this function to disable for older glibcs @@ -172,7 +164,7 @@ desirable for building Bitcoin Core release binaries." (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) + (let* ((xbinutils (cross-binutils target)) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target @@ -198,28 +190,9 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define (make-nsis-with-sde-support base-nsis) +(define (make-nsis-for-gcc-10 base-nsis) (package-with-extra-patches base-nsis - (search-our-patches "nsis-SConstruct-sde-support.patch"))) - -(define-public font-tuffy - (package - (name "font-tuffy") - (version "20120614") - (source - (origin - (method url-fetch) - (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz")) - (file-name (string-append name "-" version ".tar.gz")) - (sha256 - (base32 - "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6")))) - (build-system font-build-system) - (home-page "http://tulrich.com/fonts/") - (synopsis "The Tuffy Truetype Font Family") - (description - "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") - (license license:public-domain))) + (search-our-patches "nsis-gcc-10-memmove.patch"))) (define-public lief (package @@ -276,34 +249,6 @@ signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and thus should be able to compile on most platforms where these exist.") (license license:gpl3+))) ; license is with openssl exception -(define-public python-asn1crypto - (package - (name "python-asn1crypto") - (version "1.4.0") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/wbond/asn1crypto") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "19abibn6jw20mzi1ln4n9jjvpdka8ygm4m439hplyrdfqbvgm01r")))) - (build-system python-build-system) - (arguments - '(#:phases - (modify-phases %standard-phases - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests")))))) - (home-page "https://github.com/wbond/asn1crypto") - (synopsis "ASN.1 parser and serializer in Python") - (description "asn1crypto is an ASN.1 parser and serializer with definitions -for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, -PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") - (license license:expat))) - (define-public python-elfesteem (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) (package @@ -318,7 +263,8 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (file-name (git-file-name name commit)) (sha256 (base32 - "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")))) + "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")) + (patches (search-our-patches "elfsteem-value-error-python-39.patch")))) (build-system python-build-system) ;; There are no tests, but attempting to run python setup.py test leads to ;; PYTHONPATH problems, just disable the test @@ -390,6 +336,8 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (define-public python-oscryptotests (package (inherit python-oscrypto) (name "python-oscryptotests") + (propagated-inputs + `(("python-oscrypto" ,python-oscrypto))) (arguments `(#:tests? #f #:phases @@ -449,6 +397,11 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (string-append indent "@unittest.skip(\"Disabled by Guix\")\n" line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) #t)) (replace 'check (lambda _ @@ -461,16 +414,6 @@ certificates or paths. Supports various options, including: validation at a specific moment in time, whitelisting and revocation checks.") (license license:expat)))) -(define-public python-requests-2.25.1 - (package (inherit python-requests) - (version "2.25.1") - (source (origin - (method url-fetch) - (uri (pypi-uri "requests" version)) - (sha256 - (base32 - "015qflyqsgsz09gnar69s6ga74ivq5kch69s4qxz3904m7a3v5r7")))))) - (define-public python-altgraph (package (name "python-altgraph") @@ -563,7 +506,7 @@ and endian independent.") ("python-oscrypto" ,python-oscrypto) ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) - ("python-requests" ,python-requests-2.25.1) + ("python-requests" ,python-requests) ("python-macholib" ,python-macholib) ("libcrypto" ,openssl))) ;; There are no tests, but attempting to run python setup.py test leads to @@ -577,7 +520,7 @@ inspecting signatures in Mach-O binaries.") (define-public glibc-2.24 (package - (inherit glibc) + (inherit glibc-2.31) (version "2.24") (source (origin (method git-fetch) @@ -593,9 +536,21 @@ inspecting signatures in Mach-O binaries.") "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" "glibc-2.24-no-build-time-cxx-header-run.patch")))))) -(define glibc-2.27/bitcoin-patched - (package-with-extra-patches glibc-2.27 - (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) +(define-public glibc-2.27/bitcoin-patched + (package + (inherit glibc-2.31) + (version "2.27") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "23158b08a0908f381459f273a984c6fd328363cb"))) + (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb")) + (sha256 + (base32 + "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch")))))) (packages->manifest (append @@ -643,7 +598,7 @@ inspecting signatures in Mach-O binaries.") ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - (make-nsis-with-sde-support nsis-x86_64) + (make-nsis-for-gcc-10 nsis-x86_64) osslsigncode)) ((string-contains target "-linux-") (list (cond ((string-contains target "riscv64-") @@ -653,5 +608,5 @@ inspecting signatures in Mach-O binaries.") (else (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso python-signapple)) + (list clang-toolchain-10 binutils cmake xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch deleted file mode 100644 index 8f88eb9dfd..0000000000 --- a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch +++ /dev/null @@ -1,171 +0,0 @@ -Description: Add disable opposites to the security-related flags -Author: Stephen Kitt <skitt@debian.org> - -This patch adds "no-" variants to disable the various security flags: -"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". - ---- a/ld/emultempl/pe.em -+++ b/ld/emultempl/pe.em -@@ -259,9 +261,11 @@ - (OPTION_ENABLE_LONG_SECTION_NAMES + 1) - /* DLLCharacteristics flags. */ - #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) --#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) -+#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) -+#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) - #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) --#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) -+#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) -+#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) - #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) - #define OPTION_NO_BIND (OPTION_NO_SEH + 1) - #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) -@@ -271,6 +275,7 @@ - #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) - #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) - #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) -+#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) - - static void - gld${EMULATION_NAME}_add_options -@@ -342,8 +347,10 @@ - {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, - {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, - {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, -+ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, - {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, - {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, -+ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, - {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, - {"no-seh", no_argument, NULL, OPTION_NO_SEH}, - {"no-bind", no_argument, NULL, OPTION_NO_BIND}, -@@ -351,6 +358,7 @@ - {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, - {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, - {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, -+ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, - {NULL, no_argument, NULL, 0} - }; - -@@ -485,9 +494,12 @@ - in object files\n")); - fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ - address space layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); - fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); -+ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); - fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); - fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); -+ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); - fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); - fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ - be called in this image\n")); -@@ -862,12 +874,21 @@ - case OPTION_ENABLE_RELOC_SECTION: - pe_dll_enable_reloc_section = 1; - break; -+ case OPTION_DISABLE_RELOC_SECTION: -+ pe_dll_enable_reloc_section = 0; -+ /* fall through */ -+ case OPTION_NO_DYNAMIC_BASE: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; -+ break; - case OPTION_FORCE_INTEGRITY: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; - break; - case OPTION_NX_COMPAT: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; - break; -+ case OPTION_NO_NX_COMPAT: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; -+ break; - case OPTION_NO_ISOLATION: - pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; - break; ---- a/ld/emultempl/pep.em -+++ b/ld/emultempl/pep.em -@@ -237,9 +240,12 @@ - OPTION_ENABLE_LONG_SECTION_NAMES, - OPTION_DISABLE_LONG_SECTION_NAMES, - OPTION_HIGH_ENTROPY_VA, -+ OPTION_NO_HIGH_ENTROPY_VA, - OPTION_DYNAMIC_BASE, -+ OPTION_NO_DYNAMIC_BASE, - OPTION_FORCE_INTEGRITY, - OPTION_NX_COMPAT, -+ OPTION_NO_NX_COMPAT, - OPTION_NO_ISOLATION, - OPTION_NO_SEH, - OPTION_NO_BIND, -@@ -248,7 +254,8 @@ - OPTION_NO_INSERT_TIMESTAMP, - OPTION_TERMINAL_SERVER_AWARE, - OPTION_BUILD_ID, -- OPTION_ENABLE_RELOC_SECTION -+ OPTION_ENABLE_RELOC_SECTION, -+ OPTION_DISABLE_RELOC_SECTION - }; - - static void -@@ -315,9 +322,12 @@ - {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, - {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, - {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, -+ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, - {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, -+ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, - {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, - {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, -+ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, - {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, - {"no-seh", no_argument, NULL, OPTION_NO_SEH}, - {"no-bind", no_argument, NULL, OPTION_NO_BIND}, -@@ -327,6 +337,7 @@ - {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, - {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, - {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, -+ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, - {NULL, no_argument, NULL, 0} - }; - -@@ -448,11 +461,15 @@ - in object files\n")); - fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ - layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); - fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ - address space layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); - fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); -+ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); - fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); - fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); -+ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); - fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); - fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ - be called in this image\n")); -@@ -809,12 +826,24 @@ - case OPTION_ENABLE_RELOC_SECTION: - pep_dll_enable_reloc_section = 1; - break; -+ case OPTION_DISABLE_RELOC_SECTION: -+ pep_dll_enable_reloc_section = 0; -+ /* fall through */ -+ case OPTION_NO_DYNAMIC_BASE: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; -+ /* fall through */ -+ case OPTION_NO_HIGH_ENTROPY_VA: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; -+ break; - case OPTION_FORCE_INTEGRITY: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; - break; - case OPTION_NX_COMPAT: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; - break; -+ case OPTION_NO_NX_COMPAT: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; -+ break; - case OPTION_NO_ISOLATION: - pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; - break; diff --git a/contrib/guix/patches/elfsteem-value-error-python-39.patch b/contrib/guix/patches/elfsteem-value-error-python-39.patch new file mode 100644 index 0000000000..21e1228afd --- /dev/null +++ b/contrib/guix/patches/elfsteem-value-error-python-39.patch @@ -0,0 +1,13 @@ +diff --git a/examples/otool.py b/examples/otool.py +index 2b8efc0..d797b2e 100755 +--- a/examples/otool.py ++++ b/examples/otool.py +@@ -342,7 +342,7 @@ if __name__ == '__main__': + try: + e = macho_init.MACHO(raw, + parseSymbols = False) +- except ValueError, err: ++ except ValueError as err: + print("%s:" %file) + print(" %s" % err) + continue diff --git a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch deleted file mode 100644 index f327c464f3..0000000000 --- a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch +++ /dev/null @@ -1,400 +0,0 @@ -guix: repro: Sort find output in libtool for gcc-8 - -Otherwise the resulting .a static libraries (e.g. libstdc++.a) will not -be reproducible and end up making the Bitcoin binaries non-reproducible -as well. - -See: https://reproducible-builds.org/docs/archives/#gnu-libtool - -diff --git a/gcc/configure b/gcc/configure -index 97ba7d7d69c..e37a96f0c0c 100755 ---- a/gcc/configure -+++ b/gcc/configure -@@ -19720,20 +19720,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libcc1/configure b/libcc1/configure -index f53a121611c..5740ca90cab 100755 ---- a/libcc1/configure -+++ b/libcc1/configure -@@ -12221,20 +12221,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libffi/configure b/libffi/configure -index 790a291011f..54b1ac18306 100755 ---- a/libffi/configure -+++ b/libffi/configure -@@ -12661,20 +12661,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libgo/config/libtool.m4 b/libgo/config/libtool.m4 -index f7005947454..8a84417b828 100644 ---- a/libgo/config/libtool.m4 -+++ b/libgo/config/libtool.m4 -@@ -6010,20 +6010,20 @@ if test "$_lt_caught_CXX_error" != yes; then - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libgo/config/ltmain.sh b/libgo/config/ltmain.sh -index ce66b44906a..0f81c401407 100644 ---- a/libgo/config/ltmain.sh -+++ b/libgo/config/ltmain.sh -@@ -2917,7 +2917,7 @@ func_extract_archives () - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do -- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` -+ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ -@@ -2932,7 +2932,7 @@ func_extract_archives () - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac -- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` -+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result="$my_oldobjs" -diff --git a/libhsail-rt/configure b/libhsail-rt/configure -index a4fcc10c1f9..8e671229fcd 100755 ---- a/libhsail-rt/configure -+++ b/libhsail-rt/configure -@@ -12244,20 +12244,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libitm/configure b/libitm/configure -index dbf386db434..29d4f10611f 100644 ---- a/libitm/configure -+++ b/libitm/configure -@@ -13067,20 +13067,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/liboffloadmic/configure b/liboffloadmic/configure -index f873716991b..7aa9186b10e 100644 ---- a/liboffloadmic/configure -+++ b/liboffloadmic/configure -@@ -12379,20 +12379,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/liboffloadmic/plugin/configure b/liboffloadmic/plugin/configure -index c031eb3e7fa..67fc7368f21 100644 ---- a/liboffloadmic/plugin/configure -+++ b/liboffloadmic/plugin/configure -@@ -12086,20 +12086,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libsanitizer/configure b/libsanitizer/configure -index 4695bc7d4f7..cb7d25c07e6 100755 ---- a/libsanitizer/configure -+++ b/libsanitizer/configure -@@ -13308,20 +13308,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libstdc++-v3/configure b/libstdc++-v3/configure -index 61457e940ec..21ef1f61e41 100755 ---- a/libstdc++-v3/configure -+++ b/libstdc++-v3/configure -@@ -13087,20 +13087,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libtool.m4 b/libtool.m4 -index 24d13f34409..940faaa161d 100644 ---- a/libtool.m4 -+++ b/libtool.m4 -@@ -6005,20 +6005,20 @@ if test "$_lt_caught_CXX_error" != yes; then - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libvtv/configure b/libvtv/configure -index a197f750453..31ab3a0637b 100755 ---- a/libvtv/configure -+++ b/libvtv/configure -@@ -13339,20 +13339,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/ltmain.sh b/ltmain.sh -index 9503ec85d70..79f9ba89af5 100644 ---- a/ltmain.sh -+++ b/ltmain.sh -@@ -2917,7 +2917,7 @@ func_extract_archives () - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do -- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` -+ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ -@@ -2932,7 +2932,7 @@ func_extract_archives () - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac -- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` -+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result="$my_oldobjs" diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch deleted file mode 100644 index f58406a7a0..0000000000 --- a/contrib/guix/patches/nsis-SConstruct-sde-support.patch +++ /dev/null @@ -1,18 +0,0 @@ -https://github.com/kichik/nsis/pull/13 -https://sourceforge.net/p/nsis/code/7248/ - -diff --git a/SConstruct b/SConstruct -index e8252c9..41786f2 100755 ---- a/SConstruct -+++ b/SConstruct -@@ -95,8 +95,8 @@ default_doctype = 'html' - if defenv.WhereIs('hhc', os.environ['PATH']): - default_doctype = 'chm' - --from time import strftime, gmtime --cvs_version = strftime('%d-%b-%Y.cvs', gmtime()) -+import time -+cvs_version = time.strftime('%d-%b-%Y.cvs', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) - - opts = Variables() - diff --git a/contrib/guix/patches/nsis-gcc-10-memmove.patch b/contrib/guix/patches/nsis-gcc-10-memmove.patch new file mode 100644 index 0000000000..a1aadfd4f3 --- /dev/null +++ b/contrib/guix/patches/nsis-gcc-10-memmove.patch @@ -0,0 +1,23 @@ +commit f6df41524e703dc471e283e566a48e05a735b7f2 +Author: Anders <anders_k@users.sourceforge.net> +Date: Sat Jun 27 23:18:45 2020 +0000 + + Don't let GCC 10 generate memmove calls (bug #1248) + + git-svn-id: https://svn.code.sf.net/p/nsis/code/NSIS/trunk@7189 212acab6-be3b-0410-9dea-997c60f758d6 + +diff --git a/SCons/Config/gnu b/SCons/Config/gnu +index bfcb362d..21fa446b 100644 +--- a/SCons/Config/gnu ++++ b/SCons/Config/gnu +@@ -103,6 +103,10 @@ stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries + stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align + stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file + ++conf = FlagsConfigure(stub_env) ++conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) ++conf.Finish() ++ + stub_uenv = stub_env.Clone() + stub_uenv.Append(CPPDEFINES = ['_UNICODE', 'UNICODE']) + diff --git a/contrib/macdeploy/background.svg b/contrib/macdeploy/background.svg deleted file mode 100644 index 9c330af451..0000000000 --- a/contrib/macdeploy/background.svg +++ /dev/null @@ -1,34 +0,0 @@ -<?xml version="1.0" standalone="no"?> -<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN" - "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"> -<svg version="1.0" xmlns="http://www.w3.org/2000/svg" width="1000pt" height="640pt" viewBox="0 0 1000 640" preserveAspectRatio="xMidYMid meet"> - <!-- kate: space-indent off; - Copyright (c) 2015 The Bitcoin Core developers - Distributed under the MIT software license, see the accompanying - file COPYING or http://www.opensource.org/licenses/mit-license.php. - --> - <style type="text/css"><![CDATA[ - text { - font-family: "Tuffy"; - font-size: 86px; - fill: gray; - text-anchor: middle; - } - ]]></style> - <defs> - <linearGradient id="gradient" x1="0%" y1="0%" x2="100%" y2="100%"> - <stop offset="0%" style="stop-color:rgb(239,239,239);stop-opacity:1" /> - <stop offset="33%" style="stop-color:rgb(239,239,239);stop-opacity:1" /> - <stop offset="80%" style="stop-color:rgb(205,205,205);stop-opacity:1" /> - <stop offset="100%" style="stop-color:rgb(204,204,204);stop-opacity:1" /> - </linearGradient> - </defs> - <rect width="1000" height="640" style="fill:url(#gradient);stroke-width:0" /> - <g transform="translate(500,0) scale(0.9, 1)"> - <text x="0" y="114">PACKAGE_NAME</text> - </g> - <g transform="translate(0.000000,640.000000) scale(0.100000,-0.100000)" - fill="#000000" stroke="none"> - <path d="M4995 3705 c-24 -23 -25 -29 -25 -165 l0 -140 -306 0 -306 0 -29 -29 c-29 -29 -29 -31 -29 -141 0 -110 0 -112 29 -141 l29 -29 306 0 306 0 0 -140 c0 -136 1 -142 25 -165 16 -17 35 -25 57 -25 29 0 72 32 306 226 180 149 274 233 278 250 13 53 -2 70 -278 299 -235 194 -277 225 -306 225 -22 0 -41 -8 -57 -25z" fixlter="url(#glow)"/> - </g> -</svg> diff --git a/contrib/macdeploy/background.tiff b/contrib/macdeploy/background.tiff Binary files differnew file mode 100644 index 0000000000..1fb088c837 --- /dev/null +++ b/contrib/macdeploy/background.tiff diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 055a932eee..3b76108034 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -575,7 +575,7 @@ if config.dmg is not None: os.mkdir(os.path.dirname(bg_path)) if verbose: print('background.tiff', "->", bg_path) - shutil.copy2('background.tiff', bg_path) + shutil.copy2('contrib/macdeploy/background.tiff', bg_path) os.symlink("/Applications", os.path.join(disk_root, "Applications")) diff --git a/depends/Makefile b/depends/Makefile index 67adc6cc16..73e2af5501 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -39,6 +39,7 @@ NO_SQLITE ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= +NO_USDT ?= NO_NATPMP ?= MULTIPROCESS ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources @@ -136,7 +137,7 @@ include packages/packages.mk build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') $(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') -qrencode_packages_$(NO_QR) = $(qrencode_packages) +qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) @@ -149,8 +150,9 @@ natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) +usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -228,6 +230,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_bdb@|$(NO_BDB)|' \ -e 's|@no_sqlite@|$(NO_SQLITE)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ + -e 's|@no_usdt@|$(NO_USDT)|' \ -e 's|@no_natpmp@|$(NO_NATPMP)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@debug@|$(DEBUG)|' \ diff --git a/depends/README.md b/depends/README.md index b851ade9c7..9f0b60adf8 100644 --- a/depends/README.md +++ b/depends/README.md @@ -46,7 +46,7 @@ The paths are automatically configured and no other options are needed unless ta #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libz-dev python3-setuptools libtinfo5 xorriso + sudo apt-get install curl bsdmainutils cmake libz-dev python3-setuptools libtinfo5 xorriso Note: You must obtain the macOS SDK before proceeding with a cross-compile. Under the depends directory, create a subdirectory named `SDKs`. diff --git a/depends/config.site.in b/depends/config.site.in index ed1a1f8d3d..95e6ae85cf 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -70,6 +70,10 @@ if test -z "$enable_zmq" && test -n "@no_zmq@"; then enable_zmq=no fi +if test -z "$enable_usdt" && test -n "@no_usdt@"; then + enable_usdt=no +fi + if test "@host_os@" = darwin; then BREW=no fi diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index be5fec570c..92fd1b81bf 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,3 +1,7 @@ +ifneq ($(shell which $(host)-g++-posix),) +mingw32_CXX := $(host)-g++-posix +endif + mingw32_CFLAGS=-pipe mingw32_CXXFLAGS=$(mingw32_CFLAGS) diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 99a7ee2fbd..fa30e80f5c 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -8,7 +8,7 @@ $(package)_dependencies=xcb_proto libXau define $(package)_set_vars $(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen --without-launchd $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -# Disable uneeded extensions. +# Disable unneeded extensions. # More info is available from: https://doc.qt.io/qt-5.15/linux-requirements.html $(package)_config_opts += --disable-composite --disable-damage --disable-dpms $(package)_config_opts += --disable-dri2 --disable-dri3 --disable-glx diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 426fa2e748..4c66b3bdb9 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,10 +1,12 @@ packages:=boost libevent -qrencode_packages = qrencode +qrencode_linux_packages = qrencode +qrencode_android_packages = qrencode +qrencode_darwin_packages = qrencode +qrencode_mingw32_packages = qrencode qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm qt_android_packages=qt - qt_darwin_packages=qt qt_mingw32_packages=qt @@ -19,6 +21,8 @@ natpmp_packages=libnatpmp multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp +usdt_linux_packages=systemtap + darwin_native_packages = native_ds_store native_mac_alias $(host_arch)_$(host_os)_native_packages += native_b2 diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk new file mode 100644 index 0000000000..833e75b978 --- /dev/null +++ b/depends/packages/systemtap.mk @@ -0,0 +1,12 @@ +package=systemtap +$(package)_version=4.5 +$(package)_download_path=https://sourceware.org/systemtap/ftp/releases/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=75078ed37e0dd2a769c9d1f9394170b2d9f4d7daa425f43ca80c13bad6cfc925 +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ + mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ + cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h +endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 9798248c61..94e92431a2 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,9 +1,9 @@ package=zeromq -$(package)_version=4.3.1 +$(package)_version=4.3.4 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb -$(package)_patches=remove_libstd_link.patch +$(package)_sha256_hash=c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5 +$(package)_patches=remove_libstd_link.patch netbsd_kevent_void.patch define $(package)_set_vars $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf @@ -17,10 +17,12 @@ endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch && \ cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds + ./autogen.sh && \ $($(package)_autoconf) endef diff --git a/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch new file mode 100644 index 0000000000..eae0cf72d6 --- /dev/null +++ b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch @@ -0,0 +1,31 @@ +commit b92d4c121486f3c6e8a2cea537c53eb09894479a +Author: 0xb10c <0xb10c@gmail.com> +Date: Tue Dec 7 11:02:07 2021 +0100 + + Remove _SDT_ASM_SECTION_AUTOGROUP_SUPPORT check + + We assume that the assembler supports "?" in .pushsection directives. + This enables us to skip configure and make. + + See https://github.com/bitcoin/bitcoin/issues/23297. + +diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h +index 97766e710..352b4ee25 100644 +--- a/includes/sys/sdt.h ++++ b/includes/sys/sdt.h +@@ -230,12 +230,10 @@ __extension__ extern unsigned long long __sdt_unsp; + nice with code in COMDAT sections, which comes up in C++ code. + Without that assembler support, some combinations of probe placements + in certain kinds of C++ code may produce link-time errors. */ +-#include "sdt-config.h" +-#if _SDT_ASM_SECTION_AUTOGROUP_SUPPORT ++/* PATCH: We assume that the assembler supports the feature. This ++ enables us to skip configure and make. In turn, this means we ++ require fewer dependencies and have shorter depend build times. */ + # define _SDT_ASM_AUTOGROUP "?" +-#else +-# define _SDT_ASM_AUTOGROUP "" +-#endif + + #define _SDT_ASM_BODY(provider, name, pack_args, args) \ + _SDT_ASM_1(990: _SDT_NOP) \ diff --git a/depends/patches/zeromq/netbsd_kevent_void.patch b/depends/patches/zeromq/netbsd_kevent_void.patch new file mode 100644 index 0000000000..4e36a363fb --- /dev/null +++ b/depends/patches/zeromq/netbsd_kevent_void.patch @@ -0,0 +1,57 @@ +commit 129137d5182967dbfcfec66bad843df2a992a78f +Author: fanquake <fanquake@gmail.com> +Date: Mon Jan 3 20:13:33 2022 +0800 + + problem: kevent udata is now void* on NetBSD Current (10) + + solution: check for the intptr_t variant in configure. + +diff --git a/configure.ac b/configure.ac +index 1a571291..402f8b86 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -308,6 +308,27 @@ case "${host_os}" in + if test "x$libzmq_netbsd_has_atomic" = "xno"; then + AC_DEFINE(ZMQ_FORCE_MUTEXES, 1, [Force to use mutexes]) + fi ++ # NetBSD Current (to become 10) has changed the type of udata in it's ++ # kevent struct from intptr_t to void * to align with darwin and other ++ # BSDs, see upstream commit: ++ # https://github.com/NetBSD/src/commit/e5ead823eb916b56589d2c6c560dbcfe4a2d0afc ++ AC_MSG_CHECKING([whether kevent udata type is intptr_t]) ++ AC_LANG_PUSH([C++]) ++ AC_LINK_IFELSE([AC_LANG_PROGRAM( ++ [[#include <sys/types.h> ++ #include <sys/event.h> ++ #include <sys/time.h>]], ++ [[struct kevent ev; ++ intptr_t udata; ++ EV_SET(&ev, 0, 0, EV_ADD, 0, 0, udata); ++ return 0;]])], ++ [libzmq_netbsd_kevent_udata_intptr_t=yes], ++ [libzmq_netbsd_kevent_udata_intptr_t=no]) ++ AC_LANG_POP([C++]) ++ AC_MSG_RESULT([$libzmq_netbsd_kevent_udata_intptr_t]) ++ if test "x$libzmq_netbsd_kevent_udata_intptr_t" = "xyes"; then ++ AC_DEFINE(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T, 1, [kevent udata type is intptr_t]) ++ fi + ;; + *openbsd*|*bitrig*) + # Define on OpenBSD to enable all library features +diff --git a/src/kqueue.cpp b/src/kqueue.cpp +index 53d82ac4..a6a7a7f2 100644 +--- a/src/kqueue.cpp ++++ b/src/kqueue.cpp +@@ -46,9 +46,9 @@ + #include "i_poll_events.hpp" + #include "likely.hpp" + +-// NetBSD defines (struct kevent).udata as intptr_t, everyone else +-// as void *. +-#if defined ZMQ_HAVE_NETBSD ++// NetBSD up to version 9 defines (struct kevent).udata as intptr_t, ++// everyone else as void *. ++#if defined ZMQ_HAVE_NETBSD && defined(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T) + #define kevent_udata_t intptr_t + #else + #define kevent_udata_t void * diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 6e54f67edc..275b7ce124 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -25,8 +25,8 @@ See [dependencies.md](dependencies.md) for a complete overview. **Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is part of the base image, and while building it is necessary to make sure that this compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CC_FOR_BUILD=cc CXX=c++` to configuration commands. Mixing different -compilers within the same executable will result in errors. +`CC=cc CXX=c++` to configuration commands. Mixing different compilers within +the same executable will result in errors. ### Building BerkeleyDB @@ -84,7 +84,7 @@ To configure with wallet: To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake +./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CXX=c++ MAKE=gmake ``` To configure with GUI: diff --git a/doc/build-osx.md b/doc/build-osx.md index 467feff410..16dc224aed 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -35,7 +35,6 @@ The following dependencies are **optional** packages required for deploying: Library | Purpose | Description ----------------------------------------------------|------------------|---------------------- -[librsvg](https://formulae.brew.sh/formula/librsvg) | Deploy Dependency| Library to render SVG files [ds_store](https://pypi.org/project/ds-store/) | Deploy Dependency| Examine and modify .DS_Store files [mac_alias](https://pypi.org/project/mac-alias/) | Deploy Dependency| Generate/Read binary alias and bookmark records @@ -219,10 +218,6 @@ This command depends on a couple of python packages, so it is required that you Ensuring that `python` is installed, you can install the deploy dependencies by running the following commands in your terminal: ``` bash -brew install librsvg -``` - -``` bash pip3 install ds_store mac_alias ``` diff --git a/doc/build-windows.md b/doc/build-windows.md index 0b895eadfb..657865795c 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -51,27 +51,6 @@ The first step is to install the mingw-w64 cross-compilation tool chain: sudo apt install g++-mingw-w64-x86-64 -Next, set the default `mingw32 g++` compiler option to POSIX<sup>[1](#footnote1)</sup>: - -``` -sudo update-alternatives --config x86_64-w64-mingw32-g++ -``` - -After running the above command, you should see output similar to that below. -Choose the option that ends with `posix`. - -``` -There are 2 choices for the alternative x86_64-w64-mingw32-g++ (providing /usr/bin/x86_64-w64-mingw32-g++). - - Selection Path Priority Status ------------------------------------------------------------- - 0 /usr/bin/x86_64-w64-mingw32-g++-win32 60 auto mode -* 1 /usr/bin/x86_64-w64-mingw32-g++-posix 30 manual mode - 2 /usr/bin/x86_64-w64-mingw32-g++-win32 60 manual mode - -Press <enter> to keep the current choice[*], or type selection number: -``` - Once the toolchain is installed the build steps are common: Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for @@ -112,13 +91,3 @@ way. This will install to `c:\workspace\bitcoin`, for example: You can also create an installer using: make deploy - -Footnotes ---------- - -<a name="footnote1">1</a>: Starting from Ubuntu Xenial 16.04, both the 32 and 64 bit Mingw-w64 packages install two different -compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more -efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers -required to support win32 threads conflict with some of the classes in the C++11 standard library, in particular std::mutex. -It's not possible to build the Bitcoin Core code using the win32 version of the Mingw-w64 cross compilers (at least not without -modifying headers in the Bitcoin Core source code). diff --git a/doc/dependencies.md b/doc/dependencies.md index 24422f1d7b..63315cdcc2 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -16,15 +16,14 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| librsvg | | | | | | | MiniUPnPc | [2.2.2](https://miniupnp.tuxfamily.org/files) | | No | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | Python (tests) | | [3.6](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.12.11](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | +| Qt | [5.15.2](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | | SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| systemtap ([tracing](tracing.md))| | | | | | +| systemtap ([tracing](tracing.md))| [4.5](https://sourceware.org/systemtap/ftp/releases/) | | | | | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | | zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | @@ -46,5 +45,4 @@ Some dependencies are not needed in all configurations. The following are some f * ZeroMQ is needed only with the `--with-zmq` option. #### Other -* librsvg is only needed if you need to run `make deploy` on (cross-compilation to) macOS. * Not-Qt-bundled zlib is required to build the [DMG tool](../contrib/macdeploy/README.md#deterministic-macos-dmg-notes) from the libdmg-hfsplus project. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 73d04837f1..9abfbc9213 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -71,6 +71,15 @@ block^@M-^?M-^?M-^?M-^?M-^?nM-^?M-^? In this case the fuzzer managed to create a `block` message which when passed to `ProcessMessage(...)` increased coverage. +It is possible to specify `bitcoind` arguments to the `fuzz` executable. +Depending on the test, they may be ignored or consumed and alter the behavior +of the test. Just make sure to use double-dash to distinguish them from the +fuzzer's own arguments: + +```sh +$ FUZZ=address_deserialize_v2 src/test/fuzz/fuzz -runs=1 fuzz_seed_corpus/address_deserialize_v2 --checkaddrman=5 --printtoconsole=1 +``` + ## Fuzzing corpora The project's collection of seed corpora is found in the [`bitcoin-core/qa-assets`](https://github.com/bitcoin-core/qa-assets) repo. diff --git a/doc/release-notes-14707.md b/doc/release-notes-14707.md deleted file mode 100644 index b53204f788..0000000000 --- a/doc/release-notes-14707.md +++ /dev/null @@ -1,19 +0,0 @@ -Wallet `receivedby` RPCs now include coinbase transactions -------------- - -Previously, the following wallet RPCs excluded coinbase transactions: - -`getreceivedbyaddress` - -`getreceivedbylabel` - -`listreceivedbyaddress` - -`listreceivedbylabel` - -This release changes this behaviour and returns results accounting for received coins from coinbase outputs. - -A new option, `include_immature_coinbase` (default=`false`), determines whether to account for immature coinbase transactions. -Immature coinbase transactions are coinbase transactions that have 100 or fewer confirmations, and are not spendable. - -The previous behaviour can be restored using the configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a future release. diff --git a/doc/release-notes-23113.md b/doc/release-notes-23113.md deleted file mode 100644 index b0904c9d7b..0000000000 --- a/doc/release-notes-23113.md +++ /dev/null @@ -1,9 +0,0 @@ -Notable changes -=============== - -Updated RPCs ------------- - -- Both `createmultisig` and `addmultisigaddress` now include a `warnings` -field, which will show a warning if a non-legacy address type is requested -when using uncompressed public keys. diff --git a/doc/release-notes-gui-459.md b/doc/release-notes-gui-459.md deleted file mode 100644 index b590ac5d45..0000000000 --- a/doc/release-notes-gui-459.md +++ /dev/null @@ -1,4 +0,0 @@ -GUI changes ------------ - -- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. diff --git a/doc/release-notes.md b/doc/release-notes.md index 17b0ef545d..7a47d76bba 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -77,13 +77,6 @@ Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) Updated RPCs ------------ -- `upgradewallet` will now automatically flush the keypool if upgrading - from a non-HD wallet to an HD wallet, to immediately start using the - newly-generated HD keys. (#23093) - -- a new RPC `newkeypool` has been added, which will flush (entirely - clear and refill) the keypool. (#23093) - - The `validateaddress` RPC now returns an `error_locations` array for invalid addresses, with the indices of invalid character locations in the address (if known). For example, this will attempt to locate up to two Bech32 errors, and @@ -106,14 +99,6 @@ Updated RPCs - `value` - `scriptPubKey` -- `listunspent` now includes `ancestorcount`, `ancestorsize`, and - `ancestorfees` for each transaction output that is still in the mempool. - (#12677) - -- `lockunspent` now optionally takes a third parameter, `persistent`, which - causes the lock to be written persistently to the wallet database. This - allows UTXOs to remain locked even after node restarts or crashes. (#23065) - - The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`, `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)` @@ -123,6 +108,10 @@ Updated RPCs fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all fields in the `fees` object are denominated in BTC. (#22689) +- Both `createmultisig` and `addmultisigaddress` now include a `warnings` + field, which will show a warning if a non-legacy address type is requested + when using uncompressed public keys. (#23113) + New RPCs -------- @@ -167,12 +156,42 @@ Tools and Utilities Wallet ------ +- `upgradewallet` will now automatically flush the keypool if upgrading + from a non-HD wallet to an HD wallet, to immediately start using the + newly-generated HD keys. (#23093) + +- a new RPC `newkeypool` has been added, which will flush (entirely + clear and refill) the keypool. (#23093) + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) + +- `receivedby` RPCs now include coinbase transactions. Previously, the + following wallet RPCs excluded coinbase transactions: `getreceivedbyaddress`, + `getreceivedbylabel`, `listreceivedbyaddress`, `listreceivedbylabel`. This + release changes this behaviour and returns results accounting for received + coins from coinbase outputs. The previous behaviour can be restored using the + configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a + future release. (#14707) + +- A new option in the same `receivedby` RPCs, `include_immature_coinbase` + (default=`false`), determines whether to account for immature coinbase + transactions. Immature coinbase transactions are coinbase transactions that + have 100 or fewer confirmations, and are not spendable. (#14707) + GUI changes ----------- - UTXOs which are locked via the GUI are now stored persistently in the wallet database, so are not lost on node shutdown or crash. (#23065) +- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. + Low-level changes ================= diff --git a/doc/tor.md b/doc/tor.md index 8dc82ca91e..d23d8a1810 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -40,9 +40,11 @@ outgoing connections, but more is possible. -onion=ip:port Set the proxy server to use for Tor onion services. You do not need to set this if it's the same as -proxy. You can use -onion=0 to explicitly disable access to onion services. + ------------------------------------------------------------------ Note: Only the -proxy option sets the proxy for DNS requests; with -onion they will not route over Tor, so use -proxy if you have privacy concerns. + ------------------------------------------------------------------ -listen When using -proxy, listening is disabled by default. If you want to manually configure an onion service (see section 3), you'll diff --git a/src/Makefile.am b/src/Makefile.am index 4199f7e89d..0b177480c8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,7 +17,7 @@ EXTRA_LIBRARIES = BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -LIBBITCOIN_SERVER=libbitcoin_server.a +LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a @@ -61,7 +61,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_CLI) \ $(LIBBITCOIN_IPC) \ $(LIBBITCOIN_WALLET) \ @@ -240,6 +240,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/epochguard.h \ util/error.h \ + util/fastrange.h \ util/fees.h \ util/getuniquepath.h \ util/golombrice.h \ @@ -248,6 +249,7 @@ BITCOIN_CORE_H = \ util/macros.h \ util/message.h \ util/moneystr.h \ + util/overflow.h \ util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ @@ -316,9 +318,9 @@ ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) # Contains code accessing mempool and chain state that is meant to be separated # from wallet and gui code (see node/README.md). Shared code should go in # libbitcoin_common or libbitcoin_util libraries, instead. -libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -libbitcoin_server_a_SOURCES = \ +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ @@ -381,10 +383,10 @@ libbitcoin_server_a_SOURCES = \ $(BITCOIN_CORE_H) if ENABLE_WALLET -libbitcoin_server_a_SOURCES += wallet/init.cpp +libbitcoin_node_a_SOURCES += wallet/init.cpp endif if !ENABLE_WALLET -libbitcoin_server_a_SOURCES += dummywallet.cpp +libbitcoin_node_a_SOURCES += dummywallet.cpp endif if ENABLE_ZMQ @@ -669,13 +671,13 @@ bitcoind_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoind_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) +bitcoind_LDADD = $(LIBBITCOIN_NODE) $(bitcoin_bin_ldadd) bitcoin_node_SOURCES = $(bitcoin_daemon_sources) init/bitcoin-node.cpp bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) +bitcoin_node_LDADD = $(LIBBITCOIN_NODE) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 2a8e4a0aac..2feb31a9e9 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -51,7 +51,7 @@ nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 6c0e6f7c6f..f4d4641feb 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -323,7 +323,7 @@ bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS bitcoin_qt_sources += $(BITCOIN_QT_RC) endif -bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) +bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_NODE) if ENABLE_WALLET bitcoin_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif @@ -357,7 +357,7 @@ SECONDARY: $(QT_QM) $(srcdir)/qt/bitcoinstrings.cpp: FORCE @test -n $(XGETTEXT) || echo "xgettext is required for updating translations" - $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $(libbitcoin_server_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) + $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $(libbitcoin_node_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_BASE_CPP) qt/bitcoin.cpp $(BITCOIN_QT_WINDOWS_CPP) $(BITCOIN_QT_WALLET_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM) @test -n $(LUPDATE) || echo "lupdate is required for updating translations" diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index a0900f2691..b92d2cb6e2 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -44,7 +44,7 @@ endif # ENABLE_WALLET nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) -qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) $(LIBTEST_UTIL) +qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_NODE) $(LIBTEST_UTIL) if ENABLE_WALLET qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9fe2a3cf8a..801745d0c6 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -38,7 +38,7 @@ BITCOIN_TEST_SUITE = \ FUZZ_SUITE_LD_COMMON = \ $(LIBTEST_UTIL) \ $(LIBTEST_FUZZ) \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ @@ -197,7 +197,7 @@ if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif -test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ +test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 2d772f2fca..9574454fd2 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -19,7 +19,7 @@ libtest_fuzz_a_SOURCES = \ test/fuzz/util.cpp \ $(TEST_FUZZ_H) -LIBTEST_FUZZ += $(LIBBITCOIN_SERVER) +LIBTEST_FUZZ += $(LIBBITCOIN_NODE) LIBTEST_FUZZ += $(LIBBITCOIN_COMMON) LIBTEST_FUZZ += $(LIBBITCOIN_UTIL) LIBTEST_FUZZ += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 0a3b99e7d2..92cb8a5ce6 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -35,7 +35,7 @@ libtest_util_a_SOURCES = \ test/util/wallet.cpp \ $(TEST_UTIL_H) -LIBTEST_UTIL += $(LIBBITCOIN_SERVER) +LIBTEST_UTIL += $(LIBBITCOIN_NODE) LIBTEST_UTIL += $(LIBBITCOIN_COMMON) LIBTEST_UTIL += $(LIBBITCOIN_UTIL) LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/addrman.cpp b/src/addrman.cpp index 15c6f2943c..3a845b5b6e 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -930,6 +930,29 @@ std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() return {info_old, info_old.nLastTry}; } +std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& addr) +{ + AssertLockHeld(cs); + + AddrInfo* addr_info = Find(addr); + + if (!addr_info) return std::nullopt; + + if(addr_info->fInTried) { + int bucket{addr_info->GetTriedBucket(nKey, m_asmap)}; + return AddressPosition(/*tried=*/true, + /*multiplicity=*/1, + /*bucket=*/bucket, + /*position=*/addr_info->GetBucketPosition(nKey, false, bucket)); + } else { + int bucket{addr_info->GetNewBucket(nKey, m_asmap)}; + return AddressPosition(/*tried=*/false, + /*multiplicity=*/addr_info->nRefCount, + /*bucket=*/bucket, + /*position=*/addr_info->GetBucketPosition(nKey, true, bucket)); + } +} + void AddrManImpl::Check() const { AssertLockHeld(cs); @@ -1116,6 +1139,15 @@ void AddrManImpl::SetServices(const CService& addr, ServiceFlags nServices) Check(); } +std::optional<AddressPosition> AddrManImpl::FindAddressEntry(const CAddress& addr) +{ + LOCK(cs); + Check(); + auto entry = FindAddressEntry_(addr); + Check(); + return entry; +} + const std::vector<bool>& AddrManImpl::GetAsmap() const { return m_asmap; @@ -1201,3 +1233,8 @@ const std::vector<bool>& AddrMan::GetAsmap() const { return m_impl->GetAsmap(); } + +std::optional<AddressPosition> AddrMan::FindAddressEntry(const CAddress& addr) +{ + return m_impl->FindAddressEntry(addr); +} diff --git a/src/addrman.h b/src/addrman.h index 7f0936be8c..0646ef368d 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -22,6 +22,31 @@ class AddrManImpl; /** Default for -checkaddrman */ static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; +/** Test-only struct, capturing info about an address in AddrMan */ +struct AddressPosition { + // Whether the address is in the new or tried table + const bool tried; + + // Addresses in the tried table should always have a multiplicity of 1. + // Addresses in the new table can have multiplicity between 1 and + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS + const int multiplicity; + + // If the address is in the new table, the bucket and position are + // populated based on the first source who sent the address. + // In certain edge cases, this may not be where the address is currently + // located. + const int bucket; + const int position; + + bool operator==(AddressPosition other) { + return std::tie(tried, multiplicity, bucket, position) == + std::tie(other.tried, other.multiplicity, other.bucket, other.position); + } + explicit AddressPosition(bool tried_in, int multiplicity_in, int bucket_in, int position_in) + : tried{tried_in}, multiplicity{multiplicity_in}, bucket{bucket_in}, position{position_in} {} +}; + /** Stochastic address manager * * Design goals: @@ -142,6 +167,15 @@ public: void SetServices(const CService& addr, ServiceFlags nServices); const std::vector<bool>& GetAsmap() const; + + /** Test-only function + * Find the address record in AddrMan and return information about its + * position. + * @param[in] addr The address record to look up. + * @return Information about the address record in AddrMan + * or nullopt if address is not found. + */ + std::optional<AddressPosition> FindAddressEntry(const CAddress& addr); }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h index bd7caf473b..5e76f72342 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -137,9 +137,11 @@ public: void SetServices(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(!cs); + std::optional<AddressPosition> FindAddressEntry(const CAddress& addr) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + const std::vector<bool>& GetAsmap() const; - friend class AddrManTest; friend class AddrManDeterministic; private: @@ -266,6 +268,8 @@ private: std::pair<CAddress, int64_t> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + std::optional<AddressPosition> FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); + //! Consistency check, taking into account m_consistency_check_ratio. //! Will std::abort if an inconsistency is detected. void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 0bebb0cf54..f7f62dfc68 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -96,7 +96,7 @@ base_uint<BITS>& base_uint<BITS>::operator/=(const base_uint& b) while (shift >= 0) { if (num >= div) { num -= div; - pn[shift / 32] |= (1 << (shift & 31)); // set a bit of the result. + pn[shift / 32] |= (1U << (shift & 31)); // set a bit of the result. } div >>= 1; // shift back. shift--; @@ -236,7 +236,7 @@ uint32_t arith_uint256::GetCompact(bool fNegative) const nCompact >>= 8; nSize++; } - assert((nCompact & ~0x007fffff) == 0); + assert((nCompact & ~0x007fffffU) == 0); assert(nSize < 256); nCompact |= nSize << 24; nCompact |= (fNegative && (nCompact & 0x007fffff) ? 0x00800000 : 0); diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 2d94e835f0..3ca58b923e 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -16,6 +16,9 @@ static constexpr size_t NUM_SOURCES = 64; static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; +static const std::vector<bool> EMPTY_ASMAP; +static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0}; + static std::vector<CAddress> g_sources; static std::vector<std::vector<CAddress>> g_addresses; @@ -74,14 +77,14 @@ static void AddrManAdd(benchmark::Bench& bench) CreateAddresses(); bench.run([&] { - AddrMan addrman{/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0}; + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); }); } static void AddrManSelect(benchmark::Bench& bench) { - AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -93,7 +96,7 @@ static void AddrManSelect(benchmark::Bench& bench) static void AddrManGetAddr(benchmark::Bench& bench) { - AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -122,7 +125,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench) // // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in // AddrMan::Good() will still be noticeable. - AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); markSomeAsGood(addrman); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index f696396e12..d7b4228566 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -19,6 +19,8 @@ using namespace std::chrono_literals; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; +const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; + namespace { void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& benchmarkResults, const std::string& filename, const char* tpl) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 4734ac558c..609c592d20 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -11,6 +11,19 @@ #include <set> +using node::NodeContext; +using wallet::AttemptSelection; +using wallet::CInputCoin; +using wallet::COutput; +using wallet::CWallet; +using wallet::CWalletTx; +using wallet::CoinEligibilityFilter; +using wallet::CoinSelectionParams; +using wallet::CreateDummyWalletDatabase; +using wallet::OutputGroup; +using wallet::SelectCoinsBnB; +using wallet::TxStateInactive; + static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<std::unique_ptr<CWalletTx>>& wtxs) { static int nextLockTime = 0; diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 15e9a1ab78..d4b8794c6d 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -14,6 +14,12 @@ #include <optional> +using wallet::CWallet; +using wallet::CreateMockWalletDatabase; +using wallet::DBErrors; +using wallet::GetBalance; +using wallet::WALLET_FLAG_DESCRIPTORS; + static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 8602e362c8..edec883264 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -630,7 +630,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) } Coin newcoin; newcoin.out.scriptPubKey = scriptPubKey; - newcoin.out.nValue = 0; + newcoin.out.nValue = MAX_MONEY; if (prevOut.exists("amount")) { newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } @@ -669,6 +669,10 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (!fHashSingle || (i < mergedTx.vout.size())) ProduceSignature(keystore, MutableTransactionSignatureCreator(&mergedTx, i, amount, nHashType), prevPubKey, sigdata); + if (amount == MAX_MONEY && !sigdata.scriptWitness.IsNull()) { + throw std::runtime_error(strprintf("Missing amount for CTxOut with scriptPubKey=%s", HexStr(prevPubKey))); + } + UpdateInput(txin, sigdata); } diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index c896d5892d..65c37f182f 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -123,7 +123,7 @@ int main(int argc, char* argv[]) ECCVerifyHandle globalVerifyHandle; ECC_Start(); - if (!WalletTool::ExecuteWalletToolFunc(args, command->command)) { + if (!wallet::WalletTool::ExecuteWalletToolFunc(args, command->command)) { return EXIT_FAILURE; } ECC_Stop(); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 30dfbfd8cd..6432e8849d 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -30,6 +30,8 @@ #include <functional> #include <optional> +using node::NodeContext; + const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; UrlDecodeFn* const URL_DECODE = urlDecode; diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 68cc6462d0..63a9ba498f 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -24,43 +24,12 @@ static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; -// Map a value x that is uniformly distributed in the range [0, 2^64) to a -// value uniformly distributed in [0, n) by returning the upper 64 bits of -// x * n. -// -// See: https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ -static uint64_t MapIntoRange(uint64_t x, uint64_t n) -{ -#ifdef __SIZEOF_INT128__ - return (static_cast<unsigned __int128>(x) * static_cast<unsigned __int128>(n)) >> 64; -#else - // To perform the calculation on 64-bit numbers without losing the - // result to overflow, split the numbers into the most significant and - // least significant 32 bits and perform multiplication piece-wise. - // - // See: https://stackoverflow.com/a/26855440 - uint64_t x_hi = x >> 32; - uint64_t x_lo = x & 0xFFFFFFFF; - uint64_t n_hi = n >> 32; - uint64_t n_lo = n & 0xFFFFFFFF; - - uint64_t ac = x_hi * n_hi; - uint64_t ad = x_hi * n_lo; - uint64_t bc = x_lo * n_hi; - uint64_t bd = x_lo * n_lo; - - uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); - uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); - return upper64; -#endif -} - uint64_t GCSFilter::HashToRange(const Element& element) const { uint64_t hash = CSipHasher(m_params.m_siphash_k0, m_params.m_siphash_k1) .Write(element.data(), element.size()) .Finalize(); - return MapIntoRange(hash, m_F); + return FastRange64(hash, m_F); } std::vector<uint64_t> GCSFilter::BuildHashedSet(const ElementSet& elements) const diff --git a/src/chain.cpp b/src/chain.cpp index 5d182e1af8..e0c29372dd 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -4,6 +4,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> +#include <util/time.h> + +std::string CBlockFileInfo::ToString() const +{ + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); +} void CChain::SetTip(CBlockIndex *pindex) { if (pindex == nullptr) { diff --git a/src/common/bloom.cpp b/src/common/bloom.cpp index c3603b5d2a..0bb72dbcbb 100644 --- a/src/common/bloom.cpp +++ b/src/common/bloom.cpp @@ -11,6 +11,7 @@ #include <script/standard.h> #include <span.h> #include <streams.h> +#include <util/fastrange.h> #include <algorithm> #include <cmath> @@ -191,14 +192,6 @@ static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash); } - -// A replacement for x % n. This assumes that x and n are 32bit integers, and x is a uniformly random distributed 32bit value -// which should be the case for a good hash. -// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ -static inline uint32_t FastMod(uint32_t x, size_t n) { - return ((uint64_t)x * (uint64_t)n) >> 32; -} - void CRollingBloomFilter::insert(Span<const unsigned char> vKey) { if (nEntriesThisGeneration == nEntriesPerGeneration) { @@ -223,7 +216,7 @@ void CRollingBloomFilter::insert(Span<const unsigned char> vKey) uint32_t h = RollingBloomHash(n, nTweak, vKey); int bit = h & 0x3F; /* FastMod works with the upper bits of h, so it is safe to ignore that the lower bits of h are already used for bit. */ - uint32_t pos = FastMod(h, data.size()); + uint32_t pos = FastRange32(h, data.size()); /* The lowest bit of pos is ignored, and set to zero for the first bit, and to one for the second. */ data[pos & ~1] = (data[pos & ~1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration & 1)) << bit; data[pos | 1] = (data[pos | 1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration >> 1)) << bit; @@ -235,7 +228,7 @@ bool CRollingBloomFilter::contains(Span<const unsigned char> vKey) const for (int n = 0; n < nHashFuncs; n++) { uint32_t h = RollingBloomHash(n, nTweak, vKey); int bit = h & 0x3F; - uint32_t pos = FastMod(h, data.size()); + uint32_t pos = FastRange32(h, data.size()); /* If the relevant bit is not set in either data[pos & ~1] or data[pos | 1], the filter does not contain vKey */ if (!(((data[pos & ~1] | data[pos | 1]) >> bit) & 1)) { return false; diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 15cb55c3ce..d0dc61c7e6 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CUCKOOCACHE_H #define BITCOIN_CUCKOOCACHE_H +#include <util/fastrange.h> + #include <algorithm> // std::find #include <array> #include <atomic> @@ -219,13 +221,8 @@ private: * One option would be to implement the same trick the compiler uses and compute the * constants for exact division based on the size, as described in "{N}-bit Unsigned * Division via {N}-bit Multiply-Add" by Arch D. Robison in 2005. But that code is - * somewhat complicated and the result is still slower than other options: - * - * Instead we treat the 32-bit random number as a Q32 fixed-point number in the range - * [0, 1) and simply multiply it by the size. Then we just shift the result down by - * 32-bits to get our bucket number. The result has non-uniformity the same as a - * mod, but it is much faster to compute. More about this technique can be found at - * https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ . + * somewhat complicated and the result is still slower than an even simpler option: + * see the FastRange32 function in util/fastrange.h. * * The resulting non-uniformity is also more equally distributed which would be * advantageous for something like linear probing, though it shouldn't matter @@ -241,14 +238,14 @@ private: */ inline std::array<uint32_t, 8> compute_hashes(const Element& e) const { - return {{(uint32_t)(((uint64_t)hash_function.template operator()<0>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<1>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<2>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<3>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<4>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<5>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<6>(e) * (uint64_t)size) >> 32), - (uint32_t)(((uint64_t)hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}}; + return {{FastRange32(hash_function.template operator()<0>(e), size), + FastRange32(hash_function.template operator()<1>(e), size), + FastRange32(hash_function.template operator()<2>(e), size), + FastRange32(hash_function.template operator()<3>(e), size), + FastRange32(hash_function.template operator()<4>(e), size), + FastRange32(hash_function.template operator()<5>(e), size), + FastRange32(hash_function.template operator()<6>(e), size), + FastRange32(hash_function.template operator()<7>(e), size)}}; } /** invalid returns a special index that can never be inserted to diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 23e432e767..2b94ed611b 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -6,7 +6,6 @@ #include <walletinitinterface.h> class ArgsManager; -class CWallet; namespace interfaces { class Chain; @@ -21,7 +20,7 @@ public: bool HasWalletSupport() const override {return false;} void AddWalletOptions(ArgsManager& argsman) const override; bool ParameterInteraction() const override {return true;} - void Construct(NodeContext& node) const override {LogPrintf("No wallet support compiled in!\n");} + void Construct(node::NodeContext& node) const override {LogPrintf("No wallet support compiled in!\n");} }; void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const @@ -59,11 +58,6 @@ const WalletInitInterface& g_wallet_init_interface = DummyWalletInit(); namespace interfaces { -std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - std::unique_ptr<WalletLoader> MakeWalletLoader(Chain& chain, ArgsManager& args) { throw std::logic_error("Wallet function called in non-wallet build."); @@ -88,10 +88,17 @@ static inline auto quoted(const std::string& s) // Allow safe path append operations. static inline path operator+(path p1, path p2) { - p1 += std::move(p2); + p1 += static_cast<boost::filesystem::path&&>(p2); return p1; } +// Disallow implicit std::string conversion for copy_file +// to avoid locale-dependent encoding on Windows. +static inline void copy_file(const path& from, const path& to, copy_option options) +{ + boost::filesystem::copy_file(from, to, options); +} + /** * Convert path object to byte string. On POSIX, paths natively are byte * strings, so this is trivial. On Windows, paths natively are Unicode, so an diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 03328ac42c..e00c68585e 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -2,6 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <httpserver.h> #include <chainparamsbase.h> @@ -597,7 +601,13 @@ CService HTTPRequest::GetPeer() const // evhttp retains ownership over returned address string const char* address = ""; uint16_t port = 0; + +#ifdef HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR + evhttp_connection_get_peer(con, &address, &port); +#else evhttp_connection_get_peer(con, (char**)&address, &port); +#endif // HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR + peer = LookupNumeric(address, port); } return peer; diff --git a/src/index/base.cpp b/src/index/base.cpp index 8e94c33c1c..2e3d500cd1 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -14,6 +14,8 @@ #include <validation.h> // For g_chainman #include <warnings.h> +using node::ReadBlockFromDisk; + constexpr uint8_t DB_BEST_BLOCK{'B'}; constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index f931e6eb0b..4f99eddfd7 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -9,6 +9,8 @@ #include <node/blockstorage.h> #include <util/system.h> +using node::UndoReadFromDisk; + /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by * height, and those belonging to blocks that have been reorganized out of the active chain are diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 9ab9209ca4..ef247dc119 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -12,6 +12,12 @@ #include <undo.h> #include <validation.h> +using node::CCoinsStats; +using node::GetBogoSize; +using node::ReadBlockFromDisk; +using node::TxOutSer; +using node::UndoReadFromDisk; + static constexpr uint8_t DB_BLOCK_HASH{'s'}; static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; static constexpr uint8_t DB_MUHASH{'M'}; @@ -321,7 +327,7 @@ bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& co coins_stats.hashSerialized = entry.muhash; coins_stats.nTransactionOutputs = entry.transaction_output_count; coins_stats.nBogoSize = entry.bogo_size; - coins_stats.nTotalAmount = entry.total_amount; + coins_stats.total_amount = entry.total_amount; coins_stats.total_subsidy = entry.total_subsidy; coins_stats.total_unspendable_amount = entry.total_unspendable_amount; coins_stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index a575b37c7c..d2a6c9c964 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -52,7 +52,7 @@ public: explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Look up stats for a specific block using CBlockIndex - bool LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const; + bool LookUpStats(const CBlockIndex* block_index, node::CCoinsStats& coins_stats) const; }; /// The global UTXO set hash object. diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index bacaa1c16c..e9aeb58194 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -9,6 +9,8 @@ #include <util/system.h> #include <validation.h> +using node::OpenBlockFile; + constexpr uint8_t DB_TXINDEX{'t'}; std::unique_ptr<TxIndex> g_txindex; diff --git a/src/init.cpp b/src/init.cpp index b11eb762f2..015e17596c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -95,6 +95,22 @@ #include <zmq/zmqrpc.h> #endif +using node::CacheSizes; +using node::CalculateCacheSizes; +using node::ChainstateLoadVerifyError; +using node::ChainstateLoadingError; +using node::CleanupBlockRevFiles; +using node::DEFAULT_PRINTPRIORITY; +using node::DEFAULT_STOPAFTERBLOCKIMPORT; +using node::LoadChainstate; +using node::NodeContext; +using node::ThreadImport; +using node::VerifyLoadedChainstate; +using node::fHavePruned; +using node::fPruneMode; +using node::fReindex; +using node::nPruneTarget; + static const bool DEFAULT_PROXYRANDOMIZE = true; static const bool DEFAULT_REST_ENABLE = false; @@ -1404,31 +1420,31 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) uiInterface.InitMessage(_("Loading block index…").translated); const int64_t load_block_index_start_time = GetTimeMillis(); - std::optional<ChainstateLoadingError> rv; + std::optional<ChainstateLoadingError> maybe_load_error; try { - rv = LoadChainstate(fReset, - chainman, - Assert(node.mempool.get()), - fPruneMode, - chainparams.GetConsensus(), - fReindexChainState, - cache_sizes.block_tree_db, - cache_sizes.coins_db, - cache_sizes.coins, - false, - false, - ShutdownRequested, - []() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down."), - "", CClientUIInterface::MSG_ERROR); - }); + maybe_load_error = LoadChainstate(fReset, + chainman, + Assert(node.mempool.get()), + fPruneMode, + chainparams.GetConsensus(), + fReindexChainState, + cache_sizes.block_tree_db, + cache_sizes.coins_db, + cache_sizes.coins, + /*block_tree_db_in_memory=*/false, + /*coins_db_in_memory=*/false, + /*shutdown_requested=*/ShutdownRequested, + /*coins_error_cb=*/[]() { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down."), + "", CClientUIInterface::MSG_ERROR); + }); } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - rv = ChainstateLoadingError::ERROR_GENERIC_BLOCKDB_OPEN_FAILED; + maybe_load_error = ChainstateLoadingError::ERROR_GENERIC_BLOCKDB_OPEN_FAILED; } - if (rv.has_value()) { - switch (rv.value()) { + if (maybe_load_error.has_value()) { + switch (maybe_load_error.value()) { case ChainstateLoadingError::ERROR_LOADING_BLOCK_DB: strLoadError = _("Error loading block database"); break; @@ -1462,7 +1478,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) break; } } else { - std::optional<ChainstateLoadVerifyError> rv2; + std::optional<ChainstateLoadVerifyError> maybe_verify_error; try { uiInterface.InitMessage(_("Verifying blocks…").translated); auto check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); @@ -1470,19 +1486,19 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", MIN_BLOCKS_TO_KEEP); } - rv2 = VerifyLoadedChainstate(chainman, - fReset, - fReindexChainState, - chainparams.GetConsensus(), - check_blocks, - args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL), - static_cast<int64_t(*)()>(GetTime)); + maybe_verify_error = VerifyLoadedChainstate(chainman, + fReset, + fReindexChainState, + chainparams.GetConsensus(), + check_blocks, + args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL), + /*get_unix_time_seconds=*/static_cast<int64_t(*)()>(GetTime)); } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - rv2 = ChainstateLoadVerifyError::ERROR_GENERIC_FAILURE; + maybe_verify_error = ChainstateLoadVerifyError::ERROR_GENERIC_FAILURE; } - if (rv2.has_value()) { - switch (rv2.value()) { + if (maybe_verify_error.has_value()) { + switch (maybe_verify_error.value()) { case ChainstateLoadVerifyError::ERROR_BLOCK_FROM_FUTURE: strLoadError = _("The block database contains a block which appears to be from the future. " "This may be due to your computer's date and time being set incorrectly. " diff --git a/src/init.h b/src/init.h index 0956d9d5ed..1292cc1a3a 100644 --- a/src/init.h +++ b/src/init.h @@ -16,14 +16,16 @@ static constexpr bool DEFAULT_DAEMON = false; static constexpr bool DEFAULT_DAEMONWAIT = false; class ArgsManager; -struct NodeContext; namespace interfaces { struct BlockAndHeaderTipInfo; } +namespace node { +struct NodeContext; +} // namespace node /** Interrupt threads */ -void Interrupt(NodeContext& node); -void Shutdown(NodeContext& node); +void Interrupt(node::NodeContext& node); +void Shutdown(node::NodeContext& node); //!Initialize the logging infrastructure void InitLogging(const ArgsManager& args); //!Parameter interaction: change current parameters depending on various rules @@ -55,13 +57,13 @@ bool AppInitLockDataDirectory(); /** * Initialize node and wallet interface pointers. Has no prerequisites or side effects besides allocating memory. */ -bool AppInitInterfaces(NodeContext& node); +bool AppInitInterfaces(node::NodeContext& node); /** * Bitcoin core main initialization. * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info = nullptr); +bool AppInitMain(node::NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info = nullptr); /** * Register all arguments with the ArgsManager diff --git a/src/init/bitcoin-gui.cpp b/src/init/bitcoin-gui.cpp index f13bd4a05d..e297b48718 100644 --- a/src/init/bitcoin-gui.cpp +++ b/src/init/bitcoin-gui.cpp @@ -33,7 +33,7 @@ public: } std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } interfaces::Ipc* ipc() override { return m_ipc.get(); } - NodeContext m_node; + node::NodeContext m_node; std::unique_ptr<interfaces::Ipc> m_ipc; }; } // namespace diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp index be6dfefe7d..511a872bc0 100644 --- a/src/init/bitcoin-node.cpp +++ b/src/init/bitcoin-node.cpp @@ -20,7 +20,7 @@ const char* EXE_NAME = "bitcoin-node"; class BitcoinNodeInit : public interfaces::Init { public: - BitcoinNodeInit(NodeContext& node, const char* arg0) + BitcoinNodeInit(node::NodeContext& node, const char* arg0) : m_node(node), m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) { @@ -35,14 +35,14 @@ public: } std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } interfaces::Ipc* ipc() override { return m_ipc.get(); } - NodeContext& m_node; + node::NodeContext& m_node; std::unique_ptr<interfaces::Ipc> m_ipc; }; } // namespace } // namespace init namespace interfaces { -std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status) +std::unique_ptr<Init> MakeNodeInit(node::NodeContext& node, int argc, char* argv[], int& exit_status) { auto init = std::make_unique<init::BitcoinNodeInit>(node, argc > 0 ? argv[0] : ""); // Check if bitcoin-node is being invoked as an IPC server. If so, then diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp index 17a074cb8d..37d4e426c5 100644 --- a/src/init/bitcoin-qt.cpp +++ b/src/init/bitcoin-qt.cpp @@ -29,7 +29,7 @@ public: return MakeWalletLoader(chain, *Assert(m_node.args)); } std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } - NodeContext m_node; + node::NodeContext m_node; }; } // namespace } // namespace init diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp index 9c2e44569a..2addff07c1 100644 --- a/src/init/bitcoind.cpp +++ b/src/init/bitcoind.cpp @@ -12,6 +12,8 @@ #include <memory> +using node::NodeContext; + namespace init { namespace { class BitcoindInit : public interfaces::Init diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 9eb0d680e3..4f5105a5c1 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -28,7 +28,9 @@ enum class RBFTransactionState; struct bilingual_str; struct CBlockLocator; struct FeeCalculation; +namespace node { struct NodeContext; +} // namespace node namespace interfaces { @@ -316,7 +318,7 @@ public: }; //! Return implementation of Chain interface. -std::unique_ptr<Chain> MakeChain(NodeContext& node); +std::unique_ptr<Chain> MakeChain(node::NodeContext& node); } // namespace interfaces diff --git a/src/interfaces/init.h b/src/interfaces/init.h index a4ecf4b5d1..2153076366 100644 --- a/src/interfaces/init.h +++ b/src/interfaces/init.h @@ -7,7 +7,9 @@ #include <memory> +namespace node { struct NodeContext; +} // namespace node namespace interfaces { class Chain; @@ -40,7 +42,7 @@ public: //! status code to exit with. If this returns non-null, the caller can start up //! normally and use the Init object to spawn and connect to other processes //! while it is running. -std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status); +std::unique_ptr<Init> MakeNodeInit(node::NodeContext& node, int argc, char* argv[], int& exit_status); //! Return implementation of Init interface for the wallet process. std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status); diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 8143839850..9c1b196d61 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -22,7 +22,6 @@ #include <vector> class BanMan; -class CCoinControl; class CFeeRate; class CNodeStats; class Coin; @@ -32,8 +31,13 @@ class proxyType; enum class SynchronizationState; enum class TransactionError; struct CNodeStateStats; -struct NodeContext; struct bilingual_str; +namespace node { +struct NodeContext; +} // namespace node +namespace wallet { +class CCoinControl; +} // namespace wallet namespace interfaces { class Handler; @@ -242,12 +246,12 @@ public: //! Get and set internal node context. Useful for testing, but not //! accessible across processes. - virtual NodeContext* context() { return nullptr; } - virtual void setContext(NodeContext* context) { } + virtual node::NodeContext* context() { return nullptr; } + virtual void setContext(node::NodeContext* context) { } }; //! Return implementation of Node interface. -std::unique_ptr<Node> MakeNode(NodeContext& context); +std::unique_ptr<Node> MakeNode(node::NodeContext& context); //! Block tip (could be a header or not, depends on the subscribed signal). struct BlockTip { diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index c81ec30227..aa33a3c951 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -6,6 +6,7 @@ #define BITCOIN_INTERFACES_WALLET_H #include <consensus/amount.h> +#include <fs.h> #include <interfaces/chain.h> // For ChainClient #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination @@ -23,19 +24,21 @@ #include <utility> #include <vector> -class CCoinControl; class CFeeRate; class CKey; -class CWallet; enum class FeeReason; enum class OutputType; enum class TransactionError; +struct PartiallySignedTransaction; +struct bilingual_str; +namespace wallet { +class CCoinControl; +class CWallet; enum isminetype : unsigned int; struct CRecipient; -struct PartiallySignedTransaction; struct WalletContext; -struct bilingual_str; using isminefilter = std::underlying_type<isminetype>::type; +} // namespace wallet namespace interfaces { @@ -107,7 +110,7 @@ public: //! Look up address in wallet, return whether exists. virtual bool getAddress(const CTxDestination& dest, std::string* name, - isminetype* is_mine, + wallet::isminetype* is_mine, std::string* purpose) = 0; //! Get wallet address list. @@ -135,8 +138,8 @@ public: virtual void listLockedCoins(std::vector<COutPoint>& outputs) = 0; //! Create transaction. - virtual CTransactionRef createTransaction(const std::vector<CRecipient>& recipients, - const CCoinControl& coin_control, + virtual CTransactionRef createTransaction(const std::vector<wallet::CRecipient>& recipients, + const wallet::CCoinControl& coin_control, bool sign, int& change_pos, CAmount& fee, @@ -158,7 +161,7 @@ public: //! Create bump transaction. virtual bool createBumpTransaction(const uint256& txid, - const CCoinControl& coin_control, + const wallet::CCoinControl& coin_control, std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, @@ -213,19 +216,19 @@ public: virtual CAmount getBalance() = 0; //! Get available balance. - virtual CAmount getAvailableBalance(const CCoinControl& coin_control) = 0; + virtual CAmount getAvailableBalance(const wallet::CCoinControl& coin_control) = 0; //! Return whether transaction input belongs to wallet. - virtual isminetype txinIsMine(const CTxIn& txin) = 0; + virtual wallet::isminetype txinIsMine(const CTxIn& txin) = 0; //! Return whether transaction output belongs to wallet. - virtual isminetype txoutIsMine(const CTxOut& txout) = 0; + virtual wallet::isminetype txoutIsMine(const CTxOut& txout) = 0; //! Return debit amount if transaction input belongs to wallet. - virtual CAmount getDebit(const CTxIn& txin, isminefilter filter) = 0; + virtual CAmount getDebit(const CTxIn& txin, wallet::isminefilter filter) = 0; //! Return credit amount if transaction input belongs to wallet. - virtual CAmount getCredit(const CTxOut& txout, isminefilter filter) = 0; + virtual CAmount getCredit(const CTxOut& txout, wallet::isminefilter filter) = 0; //! Return AvailableCoins + LockedCoins grouped by wallet address. //! (put change in one group with wallet address) @@ -240,7 +243,7 @@ public: //! Get minimum fee. virtual CAmount getMinimumFee(unsigned int tx_bytes, - const CCoinControl& coin_control, + const wallet::CCoinControl& coin_control, int* returned_target, FeeReason* reason) = 0; @@ -307,7 +310,7 @@ public: virtual std::unique_ptr<Handler> handleCanGetAddressesChanged(CanGetAddressesChangedFn fn) = 0; //! Return pointer to internal wallet class, useful for testing. - virtual CWallet* wallet() { return nullptr; } + virtual wallet::CWallet* wallet() { return nullptr; } }; //! Wallet chain client that in addition to having chain client methods for @@ -326,7 +329,7 @@ public: virtual std::string getWalletDir() = 0; //! Restore backup wallet - virtual std::unique_ptr<Wallet> restoreWallet(const std::string& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; + virtual std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; //! Return available wallets in wallet directory. virtual std::vector<std::string> listWalletDir() = 0; @@ -341,18 +344,18 @@ public: virtual std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) = 0; //! Return pointer to internal context, useful for testing. - virtual WalletContext* context() { return nullptr; } + virtual wallet::WalletContext* context() { return nullptr; } }; //! Information about one wallet address. struct WalletAddress { CTxDestination dest; - isminetype is_mine; + wallet::isminetype is_mine; std::string name; std::string purpose; - WalletAddress(CTxDestination dest, isminetype is_mine, std::string name, std::string purpose) + WalletAddress(CTxDestination dest, wallet::isminetype is_mine, std::string name, std::string purpose) : dest(std::move(dest)), is_mine(is_mine), name(std::move(name)), purpose(std::move(purpose)) { } @@ -382,10 +385,10 @@ struct WalletBalances struct WalletTx { CTransactionRef tx; - std::vector<isminetype> txin_is_mine; - std::vector<isminetype> txout_is_mine; + std::vector<wallet::isminetype> txin_is_mine; + std::vector<wallet::isminetype> txout_is_mine; std::vector<CTxDestination> txout_address; - std::vector<isminetype> txout_address_is_mine; + std::vector<wallet::isminetype> txout_address_is_mine; CAmount credit; CAmount debit; CAmount change; @@ -420,7 +423,7 @@ struct WalletTxOut //! Return implementation of Wallet interface. This function is defined in //! dummywallet.cpp and throws if the wallet component is not compiled. -std::unique_ptr<Wallet> MakeWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet); +std::unique_ptr<Wallet> MakeWallet(wallet::WalletContext& context, const std::shared_ptr<wallet::CWallet>& wallet); //! Return implementation of ChainClient interface for a wallet loader. This //! function will be undefined in builds where ENABLE_WALLET is false. diff --git a/src/net.cpp b/src/net.cpp index 019e77fd7a..0260e14da7 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -112,9 +112,9 @@ static const uint64_t RANDOMIZER_ID_ADDRCACHE = 0x1cf2e4ddd306dda9ULL; // SHA256 // bool fDiscover = true; bool fListen = true; -RecursiveMutex cs_mapLocalHost; -std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); -static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; +Mutex g_maplocalhost_mutex; +std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(g_maplocalhost_mutex); +static bool vfLimited[NET_MAX] GUARDED_BY(g_maplocalhost_mutex) = {}; std::string strSubVersion; void CConnman::AddAddrFetch(const std::string& strDest) @@ -137,7 +137,7 @@ bool GetLocal(CService& addr, const CNetAddr *paddrPeer) int nBestScore = -1; int nBestReachability = -1; { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); for (const auto& entry : mapLocalHost) { int nScore = entry.second.nScore; @@ -193,7 +193,7 @@ CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) static int GetnScore(const CService& addr) { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); const auto it = mapLocalHost.find(addr); return (it != mapLocalHost.end()) ? it->second.nScore : 0; } @@ -264,7 +264,7 @@ bool AddLocal(const CService& addr_, int nScore) LogPrintf("AddLocal(%s,%i)\n", addr.ToString(), nScore); { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); const auto [it, is_newly_added] = mapLocalHost.emplace(addr, LocalServiceInfo()); LocalServiceInfo &info = it->second; if (is_newly_added || nScore >= info.nScore) { @@ -283,7 +283,7 @@ bool AddLocal(const CNetAddr &addr, int nScore) void RemoveLocal(const CService& addr) { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); LogPrintf("RemoveLocal(%s)\n", addr.ToString()); mapLocalHost.erase(addr); } @@ -292,13 +292,13 @@ void SetReachable(enum Network net, bool reachable) { if (net == NET_UNROUTABLE || net == NET_INTERNAL) return; - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); vfLimited[net] = !reachable; } bool IsReachable(enum Network net) { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); return !vfLimited[net]; } @@ -310,7 +310,7 @@ bool IsReachable(const CNetAddr &addr) /** vote for a local address */ bool SeenLocal(const CService& addr) { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); const auto it = mapLocalHost.find(addr); if (it == mapLocalHost.end()) return false; ++it->second.nScore; @@ -321,7 +321,7 @@ bool SeenLocal(const CService& addr) /** check whether a given address is potentially local */ bool IsLocal(const CService& addr) { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); return mapLocalHost.count(addr) > 0; } @@ -553,12 +553,14 @@ std::string ConnectionTypeAsString(ConnectionType conn_type) CService CNode::GetAddrLocal() const { - LOCK(cs_addrLocal); + AssertLockNotHeld(m_addr_local_mutex); + LOCK(m_addr_local_mutex); return addrLocal; } void CNode::SetAddrLocal(const CService& addrLocalIn) { - LOCK(cs_addrLocal); + AssertLockNotHeld(m_addr_local_mutex); + LOCK(m_addr_local_mutex); if (addrLocal.IsValid()) { error("Addr local already set for node: %i. Refusing to change from %s to %s", id, addrLocal.ToString(), addrLocalIn.ToString()); } else { @@ -595,7 +597,7 @@ void CNode::CopyStats(CNodeStats& stats) X(m_addr_name); X(nVersion); { - LOCK(cs_SubVer); + LOCK(m_subver_mutex); X(cleanSubVer); } stats.fInbound = IsInboundConn(); @@ -657,7 +659,7 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) // Store received bytes per message command // to prevent a memory DOS, only allow valid commands - auto i = mapRecvBytesPerMsgCmd.find(msg.m_command); + auto i = mapRecvBytesPerMsgCmd.find(msg.m_type); if (i == mapRecvBytesPerMsgCmd.end()) { i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); } @@ -747,7 +749,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds CNetMessage msg(std::move(vRecv)); // store command string, time, and sizes - msg.m_command = hdr.GetCommand(); + msg.m_type = hdr.GetCommand(); msg.m_time = time; msg.m_message_size = hdr.nMessageSize; msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; @@ -760,7 +762,7 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds // Check checksum and header command string if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { LogPrint(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", - SanitizeString(msg.m_command), msg.m_message_size, + SanitizeString(msg.m_type), msg.m_message_size, HexStr(Span{hash}.first(CMessageHeader::CHECKSUM_SIZE)), HexStr(hdr.pchChecksum), m_node_id); @@ -1099,10 +1101,10 @@ bool CConnman::AttemptToEvictConnection() void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { struct sockaddr_storage sockaddr; socklen_t len = sizeof(sockaddr); - SOCKET hSocket = accept(hListenSocket.socket, (struct sockaddr*)&sockaddr, &len); + auto sock = hListenSocket.sock->Accept((struct sockaddr*)&sockaddr, &len); CAddress addr; - if (hSocket == INVALID_SOCKET) { + if (!sock) { const int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK) { LogPrintf("socket error accept failed: %s\n", NetworkErrorString(nErr)); @@ -1116,15 +1118,15 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { addr = CAddress{MaybeFlipIPv6toCJDNS(addr), NODE_NONE}; } - const CAddress addr_bind{MaybeFlipIPv6toCJDNS(GetBindAddress(hSocket)), NODE_NONE}; + const CAddress addr_bind{MaybeFlipIPv6toCJDNS(GetBindAddress(sock->Get())), NODE_NONE}; NetPermissionFlags permissionFlags = NetPermissionFlags::None; hListenSocket.AddSocketPermissionFlags(permissionFlags); - CreateNodeFromAcceptedSocket(hSocket, permissionFlags, addr_bind, addr); + CreateNodeFromAcceptedSocket(std::move(sock), permissionFlags, addr_bind, addr); } -void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, +void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, NetPermissionFlags permissionFlags, const CAddress& addr_bind, const CAddress& addr) @@ -1150,27 +1152,24 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, if (!fNetworkActive) { LogPrint(BCLog::NET, "connection from %s dropped: not accepting new connections\n", addr.ToString()); - CloseSocket(hSocket); return; } - if (!IsSelectableSocket(hSocket)) + if (!IsSelectableSocket(sock->Get())) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); - CloseSocket(hSocket); return; } // According to the internet TCP_NODELAY is not carried into accepted sockets // on all platforms. Set it again here just to be sure. - SetSocketNoDelay(hSocket); + SetSocketNoDelay(sock->Get()); // Don't accept connections from banned peers. bool banned = m_banman && m_banman->IsBanned(addr); if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::NoBan) && banned) { LogPrint(BCLog::NET, "connection from %s dropped (banned)\n", addr.ToString()); - CloseSocket(hSocket); return; } @@ -1179,7 +1178,6 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::NoBan) && nInbound + 1 >= nMaxInbound && discouraged) { LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToString()); - CloseSocket(hSocket); return; } @@ -1188,7 +1186,6 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection LogPrint(BCLog::NET, "failed to find an eviction candidate - connection dropped (full)\n"); - CloseSocket(hSocket); return; } } @@ -1202,7 +1199,7 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, } const bool inbound_onion = std::find(m_onion_binds.begin(), m_onion_binds.end(), addr_bind) != m_onion_binds.end(); - CNode* pnode = new CNode(id, nodeServices, hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", ConnectionType::INBOUND, inbound_onion); + CNode* pnode = new CNode(id, nodeServices, sock->Release(), addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", ConnectionType::INBOUND, inbound_onion); pnode->AddRef(); pnode->m_permissionFlags = permissionFlags; pnode->m_prefer_evict = discouraged; @@ -1364,7 +1361,7 @@ bool CConnman::GenerateSelectSet(const std::vector<CNode*>& nodes, std::set<SOCKET>& error_set) { for (const ListenSocket& hListenSocket : vhListenSocket) { - recv_set.insert(hListenSocket.socket); + recv_set.insert(hListenSocket.sock->Get()); } for (CNode* pnode : nodes) { @@ -1646,7 +1643,7 @@ void CConnman::SocketHandlerListening(const std::set<SOCKET>& recv_set) if (interruptNet) { return; } - if (recv_set.count(listen_socket.socket) > 0) { + if (recv_set.count(listen_socket.sock->Get()) > 0) { AcceptConnection(listen_socket); } } @@ -1883,8 +1880,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) auto start = GetTime<std::chrono::microseconds>(); // Minimum time before next feeler connection (in microseconds). - auto next_feeler = PoissonNextSend(start, FEELER_INTERVAL); - auto next_extra_block_relay = PoissonNextSend(start, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); + auto next_feeler = GetExponentialRand(start, FEELER_INTERVAL); + auto next_extra_block_relay = GetExponentialRand(start, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); const bool dnsseed = gArgs.GetBoolArg("-dnsseed", DEFAULT_DNSSEED); bool add_fixed_seeds = gArgs.GetBoolArg("-fixedseeds", DEFAULT_FIXEDSEEDS); @@ -2004,7 +2001,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // // This is similar to the logic for trying extra outbound (full-relay) // peers, except: - // - we do this all the time on a poisson timer, rather than just when + // - we do this all the time on an exponential timer, rather than just when // our tip is stale // - we potentially disconnect our next-youngest block-relay-only peer, if our // newest block-relay-only peer delivers a block more recently. @@ -2013,10 +2010,10 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Because we can promote these connections to block-relay-only // connections, they do not get their own ConnectionType enum // (similar to how we deal with extra outbound peers). - next_extra_block_relay = PoissonNextSend(now, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); + next_extra_block_relay = GetExponentialRand(now, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); conn_type = ConnectionType::BLOCK_RELAY; } else if (now > next_feeler) { - next_feeler = PoissonNextSend(now, FEELER_INTERVAL); + next_feeler = GetExponentialRand(now, FEELER_INTERVAL); conn_type = ConnectionType::FEELER; fFeeler = true; } else { @@ -2335,7 +2332,7 @@ void CConnman::ThreadI2PAcceptIncoming() continue; } - CreateNodeFromAcceptedSocket(conn.sock->Release(), NetPermissionFlags::None, + CreateNodeFromAcceptedSocket(std::move(conn.sock), NetPermissionFlags::None, CAddress{conn.me, NODE_NONE}, CAddress{conn.peer, NODE_NONE}); } } @@ -2397,7 +2394,7 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, return false; } - vhListenSocket.push_back(ListenSocket(sock->Release(), permissions)); + vhListenSocket.emplace_back(std::move(sock), permissions); return true; } @@ -2706,15 +2703,6 @@ void CConnman::StopNodes() DeleteNode(pnode); } - // Close listening sockets. - for (ListenSocket& hListenSocket : vhListenSocket) { - if (hListenSocket.socket != INVALID_SOCKET) { - if (!CloseSocket(hListenSocket.socket)) { - LogPrintf("CloseSocket(hListenSocket) failed with error %s\n", NetworkErrorString(WSAGetLastError())); - } - } - } - for (CNode* pnode : m_nodes_disconnected) { DeleteNode(pnode); } @@ -3072,23 +3060,6 @@ bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) return found != nullptr && NodeFullyConnected(found) && func(found); } -std::chrono::microseconds CConnman::PoissonNextSendInbound(std::chrono::microseconds now, std::chrono::seconds average_interval) -{ - if (m_next_send_inv_to_incoming.load() < now) { - // If this function were called from multiple threads simultaneously - // it would possible that both update the next send variable, and return a different result to their caller. - // This is not possible in practice as only the net processing thread invokes this function. - m_next_send_inv_to_incoming = PoissonNextSend(now, average_interval); - } - return m_next_send_inv_to_incoming; -} - -std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, std::chrono::seconds average_interval) -{ - double unscaled = -log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */); - return now + std::chrono::duration_cast<std::chrono::microseconds>(unscaled * average_interval + 0.5us); -} - CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const { return CSipHasher(nSeed0, nSeed1).Write(id); @@ -25,6 +25,7 @@ #include <threadinterrupt.h> #include <uint256.h> #include <util/check.h> +#include <util/sock.h> #include <atomic> #include <condition_variable> @@ -229,8 +230,8 @@ struct LocalServiceInfo { uint16_t nPort; }; -extern RecursiveMutex cs_mapLocalHost; -extern std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); +extern Mutex g_maplocalhost_mutex; +extern std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(g_maplocalhost_mutex); extern const std::string NET_MESSAGE_COMMAND_OTHER; typedef std::map<std::string, uint64_t> mapMsgCmdSize; //command, total bytes @@ -277,7 +278,7 @@ public: /** Transport protocol agnostic message container. * Ideally it should only contain receive time, payload, - * command and size. + * type and size. */ class CNetMessage { public: @@ -285,7 +286,7 @@ public: std::chrono::microseconds m_time{0}; //!< time of message receipt uint32_t m_message_size{0}; //!< size of the payload uint32_t m_raw_message_size{0}; //!< used wire size of the message (including header/checksum) - std::string m_command; + std::string m_type; CNetMessage(CDataStream&& recv_in) : m_recv(std::move(recv_in)) {} @@ -433,12 +434,12 @@ public: //! Whether this peer is an inbound onion, i.e. connected via our Tor onion service. const bool m_inbound_onion; std::atomic<int> nVersion{0}; - RecursiveMutex cs_SubVer; + Mutex m_subver_mutex; /** * cleanSubVer is a sanitized string of the user agent byte array we read * from the wire. This cleaned string can safely be logged or displayed. */ - std::string cleanSubVer GUARDED_BY(cs_SubVer){}; + std::string cleanSubVer GUARDED_BY(m_subver_mutex){}; bool m_prefer_evict{false}; // This peer is preferred for eviction. bool HasPermission(NetPermissionFlags permission) const { return NetPermissions::HasFlag(m_permissionFlags, permission); @@ -617,9 +618,9 @@ public: return m_greatest_common_version; } - CService GetAddrLocal() const; + CService GetAddrLocal() const LOCKS_EXCLUDED(m_addr_local_mutex); //! May not be called more than once - void SetAddrLocal(const CService& addrLocalIn); + void SetAddrLocal(const CService& addrLocalIn) LOCKS_EXCLUDED(m_addr_local_mutex); CNode* AddRef() { @@ -692,8 +693,8 @@ private: std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread // Our address, as reported by the peer - CService addrLocal GUARDED_BY(cs_addrLocal); - mutable RecursiveMutex cs_addrLocal; + CService addrLocal GUARDED_BY(m_addr_local_mutex); + mutable Mutex m_addr_local_mutex; mapMsgCmdSize mapSendBytesPerMsgCmd GUARDED_BY(cs_vSend); mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); @@ -935,21 +936,19 @@ public: void WakeMessageHandler(); - /** Attempts to obfuscate tx time through exponentially distributed emitting. - Works assuming that a single interval is used. - Variable intervals will result in privacy decrease. - */ - std::chrono::microseconds PoissonNextSendInbound(std::chrono::microseconds now, std::chrono::seconds average_interval); - /** Return true if we should disconnect the peer for failing an inactivity check. */ bool ShouldRunInactivityChecks(const CNode& node, std::chrono::seconds now) const; private: struct ListenSocket { public: - SOCKET socket; + std::shared_ptr<Sock> sock; inline void AddSocketPermissionFlags(NetPermissionFlags& flags) const { NetPermissions::AddFlag(flags, m_permissions); } - ListenSocket(SOCKET socket_, NetPermissionFlags permissions_) : socket(socket_), m_permissions(permissions_) {} + ListenSocket(std::shared_ptr<Sock> sock_, NetPermissionFlags permissions_) + : sock{sock_}, m_permissions{permissions_} + { + } + private: NetPermissionFlags m_permissions; }; @@ -969,12 +968,12 @@ private: /** * Create a `CNode` object from a socket that has just been accepted and add the node to * the `m_nodes` member. - * @param[in] hSocket Connected socket to communicate with the peer. + * @param[in] sock Connected socket to communicate with the peer. * @param[in] permissionFlags The peer's permissions. * @param[in] addr_bind The address and port at our side of the connection. * @param[in] addr The address and port at the peer's side of the connection. */ - void CreateNodeFromAcceptedSocket(SOCKET hSocket, + void CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, NetPermissionFlags permissionFlags, const CAddress& addr_bind, const CAddress& addr); @@ -1216,8 +1215,6 @@ private: */ std::atomic_bool m_start_extra_block_relay_peers{false}; - std::atomic<std::chrono::microseconds> m_next_send_inv_to_incoming{0us}; - /** * A vector of -bind=<address>:<port>=onion arguments each of which is * an address and port that are designated for incoming Tor connections. @@ -1265,9 +1262,6 @@ private: friend struct ConnmanTestMsg; }; -/** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ -std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, std::chrono::seconds average_interval); - /** Dump binary message to file, with timestamp */ void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 51aa498fb0..cc23876af5 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -45,6 +45,12 @@ #include <optional> #include <typeinfo> +using node::ReadBlockFromDisk; +using node::ReadRawBlockFromDisk; +using node::fImporting; +using node::fPruneMode; +using node::fReindex; + /** How long to cache transactions in mapRelay for normal relay */ static constexpr auto RELAY_TX_CACHE_TIME = 15min; /** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ @@ -57,14 +63,14 @@ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1ms; * behind headers chain. */ static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; -/** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ -static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes +/** Timeout for (unprotected) outbound peers to sync to our chainwork */ +static constexpr auto CHAIN_SYNC_TIMEOUT{20min}; /** How frequently to check for stale tips */ static constexpr auto STALE_CHECK_INTERVAL{10min}; /** How frequently to check for extra outbound peers and disconnect */ static constexpr auto EXTRA_PEER_CHECK_INTERVAL{45s}; /** Minimum time an outbound-peer-eviction candidate must be connected for, in order to evict */ -static constexpr std::chrono::seconds MINIMUM_CONNECT_TIME{30}; +static constexpr auto MINIMUM_CONNECT_TIME{30s}; /** SHA256("main address relay")[0:8] */ static constexpr uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; /// Age after which a stale block will no longer be served if requested as @@ -74,7 +80,7 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; /** Time between pings automatically sent out for latency probing and keepalive */ -static constexpr std::chrono::minutes PING_INTERVAL{2}; +static constexpr auto PING_INTERVAL{2min}; /** The maximum number of entries in a locator */ static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of entries in an 'inv' protocol message */ @@ -88,19 +94,19 @@ static constexpr int32_t MAX_PEER_TX_REQUEST_IN_FLIGHT = 100; * the actual transaction (from any peer) in response to requests for them. */ static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 5000; /** How long to delay requesting transactions via txids, if we have wtxid-relaying peers */ -static constexpr auto TXID_RELAY_DELAY = std::chrono::seconds{2}; +static constexpr auto TXID_RELAY_DELAY{2s}; /** How long to delay requesting transactions from non-preferred peers */ -static constexpr auto NONPREF_PEER_TX_DELAY = std::chrono::seconds{2}; +static constexpr auto NONPREF_PEER_TX_DELAY{2s}; /** How long to delay requesting transactions from overloaded peers (see MAX_PEER_TX_REQUEST_IN_FLIGHT). */ -static constexpr auto OVERLOADED_PEER_TX_DELAY = std::chrono::seconds{2}; -/** How long to wait (in microseconds) before downloading a transaction from an additional peer */ -static constexpr std::chrono::microseconds GETDATA_TX_INTERVAL{std::chrono::seconds{60}}; +static constexpr auto OVERLOADED_PEER_TX_DELAY{2s}; +/** How long to wait before downloading a transaction from an additional peer */ +static constexpr auto GETDATA_TX_INTERVAL{60s}; /** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ static const unsigned int MAX_GETDATA_SZ = 1000; /** Number of blocks that can be requested at any given time from a single peer. */ static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; /** Time during which a peer must stall block download progress before being disconnected. */ -static constexpr auto BLOCK_STALLING_TIMEOUT = 2s; +static constexpr auto BLOCK_STALLING_TIMEOUT{2s}; /** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ static const unsigned int MAX_HEADERS_RESULTS = 2000; @@ -125,16 +131,16 @@ static const int MAX_UNCONNECTING_HEADERS = 10; /** Minimum blocks required to signal NODE_NETWORK_LIMITED */ static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; /** Average delay between local address broadcasts */ -static constexpr auto AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24h; +static constexpr auto AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24h}; /** Average delay between peer address broadcasts */ -static constexpr auto AVG_ADDRESS_BROADCAST_INTERVAL = 30s; +static constexpr auto AVG_ADDRESS_BROADCAST_INTERVAL{30s}; /** Average delay between trickled inventory transmissions for inbound peers. * Blocks and peers with NetPermissionFlags::NoBan permission bypass this. */ -static constexpr auto INBOUND_INVENTORY_BROADCAST_INTERVAL = 5s; +static constexpr auto INBOUND_INVENTORY_BROADCAST_INTERVAL{5s}; /** Average delay between trickled inventory transmissions for outbound peers. * Use a smaller delay as there is less privacy concern for them. * Blocks and peers with NetPermissionFlags::NoBan permission bypass this. */ -static constexpr auto OUTBOUND_INVENTORY_BROADCAST_INTERVAL = 2s; +static constexpr auto OUTBOUND_INVENTORY_BROADCAST_INTERVAL{2s}; /** Maximum rate of inventory items to send per second. * Limits the impact of low-fee transaction floods. */ static constexpr unsigned int INVENTORY_BROADCAST_PER_SECOND = 7; @@ -148,9 +154,9 @@ static constexpr unsigned int INVENTORY_MAX_RECENT_RELAY = 3500; * peers, and random variations in the broadcast mechanism. */ static_assert(INVENTORY_MAX_RECENT_RELAY >= INVENTORY_BROADCAST_PER_SECOND * UNCONDITIONAL_RELAY_DELAY / std::chrono::seconds{1}, "INVENTORY_RELAY_MAX too low"); /** Average delay between feefilter broadcasts in seconds. */ -static constexpr auto AVG_FEEFILTER_BROADCAST_INTERVAL = 10min; +static constexpr auto AVG_FEEFILTER_BROADCAST_INTERVAL{10min}; /** Maximum feefilter broadcast delay after significant change. */ -static constexpr auto MAX_FEEFILTER_CHANGE_DELAY = 5min; +static constexpr auto MAX_FEEFILTER_CHANGE_DELAY{5min}; /** Maximum number of compact filters that may be requested with one getcfilters. See BIP 157. */ static constexpr uint32_t MAX_GETCFILTERS_SIZE = 1000; /** Maximum number of cf hashes that may be requested with one getcfheaders. See BIP 157. */ @@ -329,7 +335,7 @@ private: EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(std::chrono::seconds now) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -444,6 +450,8 @@ private: */ std::map<NodeId, PeerRef> m_peer_map GUARDED_BY(m_peer_mutex); + std::atomic<std::chrono::microseconds> m_next_inv_to_inbounds{0us}; + /** Number of nodes with fSyncStarted. */ int nSyncStarted GUARDED_BY(cs_main) = 0; @@ -518,6 +526,15 @@ private: Mutex m_recent_confirmed_transactions_mutex; CRollingBloomFilter m_recent_confirmed_transactions GUARDED_BY(m_recent_confirmed_transactions_mutex){48'000, 0.000'001}; + /** + * For sending `inv`s to inbound peers, we use a single (exponentially + * distributed) timer for all peers. If we used a separate timer for each + * peer, a spy node could make multiple inbound connections to us to + * accurately determine when we received the transaction (and potentially + * determine the transaction's origin). */ + std::chrono::microseconds NextInvToInbounds(std::chrono::microseconds now, + std::chrono::seconds average_interval); + /** Have we requested this block from a peer */ bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -727,7 +744,7 @@ struct CNodeState { * - its chain tip has at least as much work as ours * * CHAIN_SYNC_TIMEOUT: if a peer's best known block has less work than our tip, - * set a timeout CHAIN_SYNC_TIMEOUT seconds in the future: + * set a timeout CHAIN_SYNC_TIMEOUT in the future: * - If at timeout their best known block now has more work than our tip * when the timeout was set, then either reset the timeout or clear it * (after comparing against our current tip's work) @@ -742,7 +759,7 @@ struct CNodeState { */ struct ChainSyncTimeoutState { //! A timeout used for checking whether our peer has sufficiently synced - int64_t m_timeout{0}; + std::chrono::seconds m_timeout{0s}; //! A header with the work we require on our peer's chain const CBlockIndex* m_work_header{nullptr}; //! After timeout is reached, set to true after sending getheaders @@ -819,6 +836,18 @@ static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUS nPreferredDownload += state->fPreferredDownload; } +std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::microseconds now, + std::chrono::seconds average_interval) +{ + if (m_next_inv_to_inbounds.load() < now) { + // If this function were called from multiple threads simultaneously + // it would possible that both update the next send variable, and return a different result to their caller. + // This is not possible in practice as only the net processing thread invokes this function. + m_next_inv_to_inbounds = GetExponentialRand(now, average_interval); + } + return m_next_inv_to_inbounds; +} + bool PeerManagerImpl::IsBlockRequested(const uint256& hash) { return mapBlocksInFlight.find(hash) != mapBlocksInFlight.end(); @@ -949,10 +978,10 @@ bool PeerManagerImpl::TipMayBeStale() { AssertLockHeld(cs_main); const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); - if (count_seconds(m_last_tip_update) == 0) { + if (m_last_tip_update.load() == 0s) { m_last_tip_update = GetTime<std::chrono::seconds>(); } - return count_seconds(m_last_tip_update) < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); + return m_last_tip_update.load() < GetTime<std::chrono::seconds>() - std::chrono::seconds{consensusParams.nPowTargetSpacing * 3} && mapBlocksInFlight.empty(); } bool PeerManagerImpl::CanDirectFetch() @@ -1137,7 +1166,7 @@ void PeerManagerImpl::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, // - TXID_RELAY_DELAY for txid announcements while wtxid peers are available // - OVERLOADED_PEER_TX_DELAY for announcements from peers which have at least // MAX_PEER_TX_REQUEST_IN_FLIGHT requests in flight (and don't have NetPermissionFlags::Relay). - auto delay = std::chrono::microseconds{0}; + auto delay{0us}; const bool preferred = state->fPreferredDownload; if (!preferred) delay += NONPREF_PEER_TX_DELAY; if (!gtxid.IsWtxid() && m_wtxid_relay_peers > 0) delay += TXID_RELAY_DELAY; @@ -1191,7 +1220,7 @@ void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) // Schedule next run for 10-15 minutes in the future. // We add randomness on every cycle to avoid the possibility of P2P fingerprinting. - const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); + const std::chrono::milliseconds delta = 10min + GetRandMillis(5min); scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } @@ -1296,7 +1325,7 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c // since pingtime does not update until the ping is complete, which might take a while. // So, if a ping is taking an unusually long time in flight, // the caller can immediately detect that this is happening. - std::chrono::microseconds ping_wait{0}; + auto ping_wait{0us}; if ((0 != peer->m_ping_nonce_sent) && (0 != peer->m_ping_start.load().count())) { ping_wait = GetTime<std::chrono::microseconds>() - peer->m_ping_start.load(); } @@ -1496,7 +1525,7 @@ void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) scheduler.scheduleEvery([this] { this->CheckForStaleTipAndEvictPeers(); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL}); // schedule next run for 10-15 minutes in the future - const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); + const std::chrono::milliseconds delta = 10min + GetRandMillis(5min); scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } @@ -1746,8 +1775,8 @@ void PeerManagerImpl::RelayAddress(NodeId originator, // Relay to a limited number of other nodes // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the m_addr_knowns of the chosen nodes prevent repeats - uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = m_connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24 * 60 * 60)); + const uint64_t hashAddr{addr.GetHash()}; + const CSipHasher hasher{m_connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr).Write((GetTime() + hashAddr) / (24 * 60 * 60))}; FastRandomContext insecure_rand; // Relay reachable addresses to 2 peers. Unreachable addresses are relayed randomly to 1 or 2 peers. @@ -1963,10 +1992,9 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - const std::chrono::seconds now = GetTime<std::chrono::seconds>(); + const auto now{GetTime<std::chrono::seconds>()}; // Get last mempool request time - const std::chrono::seconds mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() - : std::chrono::seconds::min(); + const auto mempool_req = pfrom.m_tx_relay != nullptr ? pfrom.m_tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process @@ -2631,7 +2659,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, pfrom.nServices = nServices; pfrom.SetAddrLocal(addrMe); { - LOCK(pfrom.cs_SubVer); + LOCK(pfrom.m_subver_mutex); pfrom.cleanSubVer = cleanSubVer; } peer->m_starting_height = starting_height; @@ -2890,7 +2918,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, int64_t nSince = nNow - 10 * 60; // Update/increment addr rate limiting bucket. - const auto current_time = GetTime<std::chrono::microseconds>(); + const auto current_time{GetTime<std::chrono::microseconds>()}; if (peer->m_addr_token_bucket < MAX_ADDR_PROCESSING_TOKEN_BUCKET) { // Don't increment bucket if it's already full const auto time_diff = std::max(current_time - peer->m_addr_token_timestamp, 0us); @@ -2976,7 +3004,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LOCK(cs_main); - const auto current_time = GetTime<std::chrono::microseconds>(); + const auto current_time{GetTime<std::chrono::microseconds>()}; uint256* best_block{nullptr}; for (CInv& inv : vInv) { @@ -3354,7 +3382,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } if (!fRejectedParents) { - const auto current_time = GetTime<std::chrono::microseconds>(); + const auto current_time{GetTime<std::chrono::microseconds>()}; for (const uint256& parent_txid : unique_parents) { // Here, we only have the txid (and not wtxid) of the @@ -4149,38 +4177,34 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt pfrom->GetId(), pfrom->m_addr_name.c_str(), pfrom->ConnectionTypeAsString().c_str(), - msg.m_command.c_str(), + msg.m_type.c_str(), msg.m_recv.size(), msg.m_recv.data() ); if (gArgs.GetBoolArg("-capturemessages", false)) { - CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /*is_incoming=*/true); + CaptureMessage(pfrom->addr, msg.m_type, MakeUCharSpan(msg.m_recv), /*is_incoming=*/true); } msg.SetVersion(pfrom->GetCommonVersion()); - const std::string& msg_type = msg.m_command; - - // Message size - unsigned int nMessageSize = msg.m_message_size; try { - ProcessMessage(*pfrom, msg_type, msg.m_recv, msg.m_time, interruptMsgProc); + ProcessMessage(*pfrom, msg.m_type, msg.m_recv, msg.m_time, interruptMsgProc); if (interruptMsgProc) return false; { LOCK(peer->m_getdata_requests_mutex); if (!peer->m_getdata_requests.empty()) fMoreWork = true; } } catch (const std::exception& e) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg_type), nMessageSize, e.what(), typeid(e).name()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg.m_type), msg.m_message_size, e.what(), typeid(e).name()); } catch (...) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(msg_type), nMessageSize); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(msg.m_type), msg.m_message_size); } return fMoreWork; } -void PeerManagerImpl::ConsiderEviction(CNode& pto, int64_t time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) { AssertLockHeld(cs_main); @@ -4195,12 +4219,12 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, int64_t time_in_seconds) // unless it's invalid, in which case we should find that out and // disconnect from them elsewhere). if (state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork) { - if (state.m_chain_sync.m_timeout != 0) { - state.m_chain_sync.m_timeout = 0; + if (state.m_chain_sync.m_timeout != 0s) { + state.m_chain_sync.m_timeout = 0s; state.m_chain_sync.m_work_header = nullptr; state.m_chain_sync.m_sent_getheaders = false; } - } else if (state.m_chain_sync.m_timeout == 0 || (state.m_chain_sync.m_work_header != nullptr && state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= state.m_chain_sync.m_work_header->nChainWork)) { + } else if (state.m_chain_sync.m_timeout == 0s || (state.m_chain_sync.m_work_header != nullptr && state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= state.m_chain_sync.m_work_header->nChainWork)) { // Our best block known by this peer is behind our tip, and we're either noticing // that for the first time, OR this peer was able to catch up to some earlier point // where we checked against our tip. @@ -4208,7 +4232,7 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, int64_t time_in_seconds) state.m_chain_sync.m_timeout = time_in_seconds + CHAIN_SYNC_TIMEOUT; state.m_chain_sync.m_work_header = m_chainman.ActiveChain().Tip(); state.m_chain_sync.m_sent_getheaders = false; - } else if (state.m_chain_sync.m_timeout > 0 && time_in_seconds > state.m_chain_sync.m_timeout) { + } else if (state.m_chain_sync.m_timeout > 0s && time_in_seconds > state.m_chain_sync.m_timeout) { // No evidence yet that our peer has synced to a chain with work equal to that // of our tip, when we first detected it was behind. Send a single getheaders // message to give the peer a chance to update us. @@ -4221,7 +4245,7 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, int64_t time_in_seconds) LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; - constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes + constexpr auto HEADERS_RESPONSE_TIME{2min}; // Bump the timeout to allow a response, which could clear the timeout // (if the response shows the peer has synced), reset the timeout (if // the peer syncs to the required work but not to our tip), or result @@ -4348,7 +4372,8 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() // Check whether our tip is stale, and if so, allow using an extra // outbound peer if (!fImporting && !fReindex && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale()) { - LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", count_seconds(now) - count_seconds(m_last_tip_update)); + LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", + count_seconds(now - m_last_tip_update.load())); m_connman.SetTryNewOutboundPeer(true); } else if (m_connman.GetTryNewOutboundPeer()) { m_connman.SetTryNewOutboundPeer(false); @@ -4428,13 +4453,13 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros FastRandomContext insecure_rand; PushAddress(peer, *local_addr, insecure_rand); } - peer.m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + peer.m_next_local_addr_send = GetExponentialRand(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // We sent an `addr` message to this peer recently. Nothing more to do. if (current_time <= peer.m_next_addr_send) return; - peer.m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); + peer.m_next_addr_send = GetExponentialRand(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); if (!Assume(peer.m_addrs_to_send.size() <= MAX_ADDR_TO_SEND)) { // Should be impossible since we always check size before adding to @@ -4506,7 +4531,7 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds c m_connman.PushMessage(&pto, CNetMsgMaker(pto.GetCommonVersion()).Make(NetMsgType::FEEFILTER, filterToSend)); pto.m_tx_relay->lastSentFeeFilter = filterToSend; } - pto.m_tx_relay->m_next_send_feefilter = PoissonNextSend(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); + pto.m_tx_relay->m_next_send_feefilter = GetExponentialRand(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); } // If the fee filter has changed substantially and it's still more than MAX_FEEFILTER_CHANGE_DELAY // until scheduled broadcast, then move the broadcast to within MAX_FEEFILTER_CHANGE_DELAY. @@ -4570,7 +4595,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // If we get here, the outgoing message serialization version is set and can't change. const CNetMsgMaker msgMaker(pto->GetCommonVersion()); - const auto current_time = GetTime<std::chrono::microseconds>(); + const auto current_time{GetTime<std::chrono::microseconds>()}; if (pto->IsAddrFetchConn() && current_time - pto->m_connected > 10 * AVG_ADDRESS_BROADCAST_INTERVAL) { LogPrint(BCLog::NET, "addrfetch connection timeout; disconnecting peer=%d\n", pto->GetId()); @@ -4786,9 +4811,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; if (pto->IsInboundConn()) { - pto->m_tx_relay->nNextInvSend = m_connman.PoissonNextSendInbound(current_time, INBOUND_INVENTORY_BROADCAST_INTERVAL); + pto->m_tx_relay->nNextInvSend = NextInvToInbounds(current_time, INBOUND_INVENTORY_BROADCAST_INTERVAL); } else { - pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, OUTBOUND_INVENTORY_BROADCAST_INTERVAL); + pto->m_tx_relay->nNextInvSend = GetExponentialRand(current_time, OUTBOUND_INVENTORY_BROADCAST_INTERVAL); } } @@ -4969,7 +4994,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Check that outbound peers have reasonable chains // GetTime() is used by this anti-DoS logic so we can test this using mocktime - ConsiderEviction(*pto, GetTime()); + ConsiderEviction(*pto, GetTime<std::chrono::seconds>()); // // Message: getdata (blocks) diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 60e874967f..cbfdcb6f11 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -21,29 +21,13 @@ #include <util/system.h> #include <validation.h> +namespace node { std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); bool fHavePruned = false; bool fPruneMode = false; uint64_t nPruneTarget = 0; -// TODO make namespace { -RecursiveMutex cs_LastBlockFile; -std::vector<CBlockFileInfo> vinfoBlockFile; -int nLastBlockFile = 0; -/** Global flag to indicate we should check to see if there are -* block/undo files that should be deleted. Set on startup -* or if we allocate more file space when we're in prune mode -*/ -bool fCheckForPruning = false; - -/** Dirty block index entries. */ -std::set<CBlockIndex*> setDirtyBlockIndex; - -/** Dirty block file entries. */ -std::set<int> setDirtyFileInfo; -// } // namespace - static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); @@ -86,7 +70,7 @@ CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block) if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) pindexBestHeader = pindexNew; - setDirtyBlockIndex.insert(pindexNew); + m_dirty_blockindex.insert(pindexNew); return pindexNew; } @@ -104,7 +88,7 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) pindex->nFile = 0; pindex->nDataPos = 0; pindex->nUndoPos = 0; - setDirtyBlockIndex.insert(pindex); + m_dirty_blockindex.insert(pindex); // Prune from m_blocks_unlinked -- any block we prune would have // to be downloaded again in order to consider its chain, at which @@ -121,8 +105,8 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) } } - vinfoBlockFile[fileNumber].SetNull(); - setDirtyFileInfo.insert(fileNumber); + m_blockfile_info[fileNumber].SetNull(); + m_dirty_fileinfo.insert(fileNumber); } void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) @@ -137,8 +121,8 @@ void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nM // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP); int count = 0; - for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { - if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { + if (m_blockfile_info[fileNumber].nSize == 0 || m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { continue; } PruneOneBlockFile(fileNumber); @@ -177,10 +161,10 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr nBuffer += nPruneTarget / 10; } - for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { - nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) { + nBytesToPrune = m_blockfile_info[fileNumber].nSize + m_blockfile_info[fileNumber].nUndoSize; - if (vinfoBlockFile[fileNumber].nSize == 0) { + if (m_blockfile_info[fileNumber].nSize == 0) { continue; } @@ -189,7 +173,7 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr } // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning - if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) { + if (m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) { continue; } @@ -290,7 +274,7 @@ bool BlockManager::LoadBlockIndex( } if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { pindex->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindex); + m_dirty_blockindex.insert(pindex); } if (pindex->IsAssumedValid() || (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && @@ -315,7 +299,7 @@ bool BlockManager::LoadBlockIndex( // because they will still be attached to the active chainstate if they // actually contain more work. // - // Instad of this height-based approach, an earlier attempt was made at + // Instead of this height-based approach, an earlier attempt was made at // detecting "holistically" whether the block index under consideration // relied on an assumed-valid ancestor, but this proved to be too slow to // be practical. @@ -348,6 +332,32 @@ void BlockManager::Unload() } m_block_index.clear(); + + m_blockfile_info.clear(); + m_last_blockfile = 0; + m_dirty_blockindex.clear(); + m_dirty_fileinfo.clear(); +} + +bool BlockManager::WriteBlockIndexDB() +{ + AssertLockHeld(::cs_main); + std::vector<std::pair<int, const CBlockFileInfo*>> vFiles; + vFiles.reserve(m_dirty_fileinfo.size()); + for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) { + vFiles.push_back(std::make_pair(*it, &m_blockfile_info[*it])); + m_dirty_fileinfo.erase(it++); + } + std::vector<const CBlockIndex*> vBlocks; + vBlocks.reserve(m_dirty_blockindex.size()); + for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) { + vBlocks.push_back(*it); + m_dirty_blockindex.erase(it++); + } + if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) { + return false; + } + return true; } bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) @@ -357,17 +367,17 @@ bool BlockManager::LoadBlockIndexDB(ChainstateManager& chainman) } // Load block file info - m_block_tree_db->ReadLastBlockFile(nLastBlockFile); - vinfoBlockFile.resize(nLastBlockFile + 1); - LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); - for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { - m_block_tree_db->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); - } - LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); - for (int nFile = nLastBlockFile + 1; true; nFile++) { + m_block_tree_db->ReadLastBlockFile(m_last_blockfile); + m_blockfile_info.resize(m_last_blockfile + 1); + LogPrintf("%s: last block file = %i\n", __func__, m_last_blockfile); + for (int nFile = 0; nFile <= m_last_blockfile; nFile++) { + m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]); + } + LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[m_last_blockfile].ToString()); + for (int nFile = m_last_blockfile + 1; true; nFile++) { CBlockFileInfo info; if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { - vinfoBlockFile.push_back(info); + m_blockfile_info.push_back(info); } else { break; } @@ -425,7 +435,7 @@ bool IsBlockPruned(const CBlockIndex* pblockindex) // If we're using -prune with -reindex, then delete block files that will be ignored by the // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile // is missing, do the same here to delete any later block files after a gap. Also delete all -// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile +// rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info // is in sync with what's actually on disk by the time we start downloading, so that pruning // works correctly. void CleanupBlockRevFiles() @@ -465,16 +475,11 @@ void CleanupBlockRevFiles() } } -std::string CBlockFileInfo::ToString() const -{ - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} - -CBlockFileInfo* GetBlockFileInfo(size_t n) +CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) { LOCK(cs_LastBlockFile); - return &vinfoBlockFile.at(n); + return &m_blockfile_info.at(n); } static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) @@ -538,32 +543,32 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) return true; } -static void FlushUndoFile(int block_file, bool finalize = false) +void BlockManager::FlushUndoFile(int block_file, bool finalize) { - FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); + FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); } } -void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) +void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) { LOCK(cs_LastBlockFile); - FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, // e.g. during IBD or a sync after a node going offline - if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); + if (!fFinalize || finalize_undo) FlushUndoFile(m_last_blockfile, finalize_undo); } -uint64_t CalculateCurrentUsage() +uint64_t BlockManager::CalculateCurrentUsage() { LOCK(cs_LastBlockFile); uint64_t retval = 0; - for (const CBlockFileInfo& file : vinfoBlockFile) { + for (const CBlockFileInfo& file : m_blockfile_info) { retval += file.nSize + file.nUndoSize; } return retval; @@ -605,44 +610,44 @@ fs::path GetBlockPosFilename(const FlatFilePos& pos) return BlockFileSeq().FileName(pos); } -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) +bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown) { LOCK(cs_LastBlockFile); - unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); + unsigned int nFile = fKnown ? pos.nFile : m_last_blockfile; + if (m_blockfile_info.size() <= nFile) { + m_blockfile_info.resize(nFile + 1); } bool finalize_undo = false; if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { + while (m_blockfile_info[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { // when the undo file is keeping up with the block file, we want to flush it explicitly // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it - finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + finalize_undo = (m_blockfile_info[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); nFile++; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); + if (m_blockfile_info.size() <= nFile) { + m_blockfile_info.resize(nFile + 1); } } pos.nFile = nFile; - pos.nPos = vinfoBlockFile[nFile].nSize; + pos.nPos = m_blockfile_info[nFile].nSize; } - if ((int)nFile != nLastBlockFile) { + if ((int)nFile != m_last_blockfile) { if (!fKnown) { - LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString()); } FlushBlockFile(!fKnown, finalize_undo); - nLastBlockFile = nFile; + m_last_blockfile = nFile; } - vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + m_blockfile_info[nFile].AddBlock(nHeight, nTime); if (fKnown) { - vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize); } else { - vinfoBlockFile[nFile].nSize += nAddSize; + m_blockfile_info[nFile].nSize += nAddSize; } if (!fKnown) { @@ -652,23 +657,23 @@ bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, return AbortNode("Disk space is too low!", _("Disk space is too low!")); } if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; + m_check_for_pruning = true; } } - setDirtyFileInfo.insert(nFile); + m_dirty_fileinfo.insert(nFile); return true; } -static bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) +bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); - pos.nPos = vinfoBlockFile[nFile].nUndoSize; - vinfoBlockFile[nFile].nUndoSize += nAddSize; - setDirtyFileInfo.insert(nFile); + pos.nPos = m_blockfile_info[nFile].nUndoSize; + m_blockfile_info[nFile].nUndoSize += nAddSize; + m_dirty_fileinfo.insert(nFile); bool out_of_space; size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); @@ -676,7 +681,7 @@ static bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; + m_check_for_pruning = true; } return true; @@ -705,7 +710,7 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } -bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { @@ -721,14 +726,14 @@ bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& st // in the block file info as below; note that this does not catch the case where the undo writes are keeping up // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in // the FindBlockPos function - if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { + if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(pindex->nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { FlushUndoFile(_pos.nFile, true); } // update nUndoPos in block index pindex->nUndoPos = _pos.nPos; pindex->nStatus |= BLOCK_HAVE_UNDO; - setDirtyBlockIndex.insert(pindex); + m_dirty_blockindex.insert(pindex); } return true; @@ -825,7 +830,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; @@ -931,3 +936,4 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile } // End scope of CImportingNow chainman.ActiveChainstate().LoadMempool(args); } +} // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index a18203f48d..78c9210892 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -29,6 +29,7 @@ namespace Consensus { struct Params; } +namespace node { static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -64,8 +65,14 @@ struct CBlockIndexWorkComparator { class BlockManager { friend CChainState; + friend ChainstateManager; private: + void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); + void FlushUndoFile(int block_file, bool finalize = false); + bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); + bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); @@ -75,9 +82,9 @@ private: * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex * (which in this case means the blockchain must be re-downloaded.) * - * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set. + * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set. * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) - * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 1000 on regtest). + * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight). * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. * A db flag records the fact that at least some block files have been pruned. @@ -86,6 +93,21 @@ private: */ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd); + RecursiveMutex cs_LastBlockFile; + std::vector<CBlockFileInfo> m_blockfile_info; + int m_last_blockfile = 0; + /** Global flag to indicate we should check to see if there are + * block/undo files that should be deleted. Set on startup + * or if we allocate more file space when we're in prune mode + */ + bool m_check_for_pruning = false; + + /** Dirty block index entries. */ + std::set<CBlockIndex*> m_dirty_blockindex; + + /** Dirty block file entries. */ + std::set<int> m_dirty_fileinfo; + public: BlockMap m_block_index GUARDED_BY(cs_main); @@ -97,12 +119,13 @@ public: std::unique_ptr<CBlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); + bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); bool LoadBlockIndexDB(ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral - * collections like setDirtyBlockIndex. + * collections like m_dirty_blockindex. */ bool LoadBlockIndex( const Consensus::Params& consensus_params, @@ -120,6 +143,16 @@ public: CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Get block file info entry for one block file */ + CBlockFileInfo* GetBlockFileInfo(size_t n); + + bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams); + + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); + + /** Calculate the amount of disk space the block & undo files currently use */ + uint64_t CalculateCurrentUsage(); + //! Returns last CBlockIndex* that is a checkpoint CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -139,12 +172,6 @@ FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos& pos); -/** Get block file info entry for one block file */ -CBlockFileInfo* GetBlockFileInfo(size_t n); - -/** Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage(); - /** * Actually unlink the specified files */ @@ -157,10 +184,8 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); -bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams); - -FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args); +} // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/caches.cpp b/src/node/caches.cpp index 36254dc714..f168332ee6 100644 --- a/src/node/caches.cpp +++ b/src/node/caches.cpp @@ -8,6 +8,7 @@ #include <util/system.h> #include <validation.h> +namespace node { CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes) { int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20); @@ -30,3 +31,4 @@ CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes) sizes.coins = nTotalCache; // the rest goes to in-memory cache return sizes; } +} // namespace node diff --git a/src/node/caches.h b/src/node/caches.h index 200f0b85b8..67388b91fd 100644 --- a/src/node/caches.h +++ b/src/node/caches.h @@ -10,6 +10,7 @@ class ArgsManager; +namespace node { struct CacheSizes { int64_t block_tree_db; int64_t coins_db; @@ -18,5 +19,6 @@ struct CacheSizes { int64_t filter_index; }; CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0); +} // namespace node #endif // BITCOIN_NODE_CACHES_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index abeebad1a6..e21116dd7e 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -8,6 +8,7 @@ #include <node/blockstorage.h> #include <validation.h> +namespace node { std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, ChainstateManager& chainman, CTxMemPool* mempool, @@ -141,7 +142,7 @@ std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManage for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { const CBlockIndex* tip = chainstate->m_chain.Tip(); - if (tip && tip->nTime > get_unix_time_seconds() + 2 * 60 * 60) { + if (tip && tip->nTime > get_unix_time_seconds() + MAX_FUTURE_BLOCK_TIME) { return ChainstateLoadVerifyError::ERROR_BLOCK_FROM_FUTURE; } @@ -156,3 +157,4 @@ std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManage return std::nullopt; } +} // namespace node diff --git a/src/node/chainstate.h b/src/node/chainstate.h index 11278a0991..279f187642 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -10,11 +10,12 @@ #include <optional> class ChainstateManager; -namespace Consensus { - struct Params; -} class CTxMemPool; +namespace Consensus { +struct Params; +} // namespace Consensus +namespace node { enum class ChainstateLoadingError { ERROR_LOADING_BLOCK_DB, ERROR_BAD_GENESIS_BLOCK, @@ -50,7 +51,7 @@ enum class ChainstateLoadingError { * this sequence, when we explicitly checked shutdown_requested() at * arbitrary points, one of those calls returned true". Therefore, a * return value other than SHUTDOWN_PROBED does not guarantee that - * shutdown_requested() hasn't been called indirectly. + * shutdown hasn't been called indirectly. * - else * - Success! */ @@ -81,5 +82,6 @@ std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManage unsigned int check_blocks, unsigned int check_level, std::function<int64_t()> get_unix_time_seconds); +} // namespace node #endif // BITCOIN_NODE_CHAINSTATE_H diff --git a/src/node/coin.cpp b/src/node/coin.cpp index 78ab46f2e8..221854c5f6 100644 --- a/src/node/coin.cpp +++ b/src/node/coin.cpp @@ -8,6 +8,7 @@ #include <txmempool.h> #include <validation.h> +namespace node { void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins) { assert(node.mempool); @@ -22,3 +23,4 @@ void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins) } } } +} // namespace node diff --git a/src/node/coin.h b/src/node/coin.h index 908850e2a5..3d534463e8 100644 --- a/src/node/coin.h +++ b/src/node/coin.h @@ -9,6 +9,8 @@ class COutPoint; class Coin; + +namespace node { struct NodeContext; /** @@ -19,6 +21,7 @@ struct NodeContext; * @param[in] node The node context to use for lookup * @param[in,out] coins map to fill */ -void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins); +void FindCoins(const node::NodeContext& node, std::map<COutPoint, Coin>& coins); +} // namespace node #endif // BITCOIN_NODE_COIN_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index b538474d8d..eed43a1bc7 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -11,11 +11,13 @@ #include <index/coinstatsindex.h> #include <serialize.h> #include <uint256.h> +#include <util/overflow.h> #include <util/system.h> #include <validation.h> #include <map> +namespace node { // Database-independent metric indicating the UTXO set size uint64_t GetBogoSize(const CScript& script_pub_key) { @@ -82,7 +84,9 @@ static void ApplyStats(CCoinsStats& stats, const uint256& hash, const std::map<u stats.nTransactions++; for (auto it = outputs.begin(); it != outputs.end(); ++it) { stats.nTransactionOutputs++; - stats.nTotalAmount += it->second.out.nValue; + if (stats.total_amount.has_value()) { + stats.total_amount = CheckedAdd(*stats.total_amount, it->second.out.nValue); + } stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey); } } @@ -95,10 +99,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& assert(pcursor); if (!pindex) { - { - LOCK(cs_main); - pindex = blockman.LookupBlockIndex(view->GetBestBlock()); - } + LOCK(cs_main); + pindex = blockman.LookupBlockIndex(view->GetBestBlock()); } stats.nHeight = Assert(pindex)->nHeight; stats.hashBlock = pindex->GetBlockHash(); @@ -180,3 +182,4 @@ static void FinalizeHash(MuHash3072& muhash, CCoinsStats& stats) stats.hashSerialized = out; } static void FinalizeHash(std::nullptr_t, CCoinsStats& stats) {} +} // namespace node diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 4a5d17b3fd..aa771b18b0 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,18 +15,22 @@ #include <cstdint> #include <functional> -class BlockManager; class CCoinsView; +namespace node { +class BlockManager; +} // namespace node +namespace node { enum class CoinStatsHashType { HASH_SERIALIZED, MUHASH, NONE, }; -struct CCoinsStats -{ - CoinStatsHashType m_hash_type; +struct CCoinsStats { + //! Which hash type to use + const CoinStatsHashType m_hash_type; + int nHeight{0}; uint256 hashBlock{}; uint64_t nTransactions{0}; @@ -34,7 +38,8 @@ struct CCoinsStats uint64_t nBogoSize{0}; uint256 hashSerialized{}; uint64_t nDiskSize{0}; - CAmount nTotalAmount{0}; + //! The total amount, or nullopt if an overflow occurred calculating it + std::optional<CAmount> total_amount{0}; //! The number of coins contained. uint64_t coins_count{0}; @@ -69,10 +74,11 @@ struct CCoinsStats }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point = {}, const CBlockIndex* pindex = nullptr); +bool GetUTXOStats(CCoinsView* view, node::BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point = {}, const CBlockIndex* pindex = nullptr); uint64_t GetBogoSize(const CScript& script_pub_key); CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin); +} // namespace node #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.cpp b/src/node/context.cpp index 9afadd09a9..893c32f1bc 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -14,5 +14,7 @@ #include <txmempool.h> #include <validation.h> +namespace node { NodeContext::NodeContext() {} NodeContext::~NodeContext() {} +} // namespace node diff --git a/src/node/context.h b/src/node/context.h index 8f5c50377d..644c997531 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -26,6 +26,7 @@ class Init; class WalletLoader; } // namespace interfaces +namespace node { //! NodeContext struct containing references to chain state and connection //! state. //! @@ -62,5 +63,6 @@ struct NodeContext { NodeContext(); ~NodeContext(); }; +} // namespace node #endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index a0ee52f8e6..1a48957f0f 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -249,8 +249,8 @@ public: bool isInitialBlockDownload() override { return chainman().ActiveChainstate().IsInitialBlockDownload(); } - bool getReindex() override { return ::fReindex; } - bool getImporting() override { return ::fImporting; } + bool getReindex() override { return node::fReindex; } + bool getImporting() override { return node::fImporting; } void setNetworkActive(bool active) override { if (m_context->connman) { @@ -649,9 +649,9 @@ public: bool havePruned() override { LOCK(cs_main); - return ::fHavePruned; + return node::fHavePruned; } - bool isReadyToBroadcast() override { return !::fImporting && !::fReindex && !isInitialBlockDownload(); } + bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } bool isInitialBlockDownload() override { return chainman().ActiveChainstate().IsInitialBlockDownload(); } @@ -729,6 +729,6 @@ public: } // namespace node namespace interfaces { -std::unique_ptr<Node> MakeNode(NodeContext& context) { return std::make_unique<node::NodeImpl>(context); } -std::unique_ptr<Chain> MakeChain(NodeContext& context) { return std::make_unique<node::ChainImpl>(context); } +std::unique_ptr<Node> MakeNode(node::NodeContext& context) { return std::make_unique<node::NodeImpl>(context); } +std::unique_ptr<Chain> MakeChain(node::NodeContext& context) { return std::make_unique<node::ChainImpl>(context); } } // namespace interfaces diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 8c3e7b7b65..7fe10ecabc 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -26,6 +26,7 @@ #include <algorithm> #include <utility> +namespace node { int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { int64_t nOldTime = pblock->nTime; @@ -464,3 +465,4 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); } +} // namespace node diff --git a/src/node/miner.h b/src/node/miner.h index 38b7b4546b..c96da874a7 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -23,6 +23,7 @@ class CScript; namespace Consensus { struct Params; }; +namespace node { static const bool DEFAULT_PRINTPRIORITY = false; struct CBlockTemplate @@ -206,5 +207,6 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman); +} // namespace node #endif // BITCOIN_NODE_MINER_H diff --git a/src/node/minisketchwrapper.cpp b/src/node/minisketchwrapper.cpp index 572df63463..67e823cb68 100644 --- a/src/node/minisketchwrapper.cpp +++ b/src/node/minisketchwrapper.cpp @@ -16,6 +16,7 @@ #include <utility> #include <vector> +namespace node { namespace { static constexpr uint32_t BITS = 32; @@ -75,3 +76,4 @@ Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits) { return Minisketch::CreateFP(BITS, Minisketch32Implementation(), max_elements, fpbits); } +} // namespace node diff --git a/src/node/minisketchwrapper.h b/src/node/minisketchwrapper.h index a8aef68d01..a92912d9ed 100644 --- a/src/node/minisketchwrapper.h +++ b/src/node/minisketchwrapper.h @@ -10,9 +10,11 @@ #include <cstddef> #include <cstdint> +namespace node { /** Wrapper around Minisketch::Minisketch(32, implementation, capacity). */ Minisketch MakeMinisketch32(size_t capacity); /** Wrapper around Minisketch::CreateFP. */ Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits); +} // namespace node #endif // BITCOIN_NODE_MINISKETCHWRAPPER_H diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 26023d5a4c..5a932f435d 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -12,6 +12,7 @@ #include <numeric> +namespace node { PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) { // Go through each input and build status @@ -147,3 +148,4 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) return result; } +} // namespace node diff --git a/src/node/psbt.h b/src/node/psbt.h index 43fe293ad0..cbb3bd8201 100644 --- a/src/node/psbt.h +++ b/src/node/psbt.h @@ -9,6 +9,7 @@ #include <optional> +namespace node { /** * Holds an analysis of one input from a PSBT */ @@ -52,5 +53,6 @@ struct PSBTAnalysis { * @return A PSBTAnalysis with information about the provided PSBT. */ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx); +} // namespace node #endif // BITCOIN_NODE_PSBT_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index d5681a0663..c7c8493f0c 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -16,6 +16,7 @@ #include <future> +namespace node { static TransactionError HandleATMPError(const TxValidationState& state, std::string& err_string_out) { err_string_out = state.ToString(); @@ -153,3 +154,4 @@ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMe } return nullptr; } +} // namespace node diff --git a/src/node/transaction.h b/src/node/transaction.h index 79f02958da..b7cf225636 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -12,11 +12,13 @@ class CBlockIndex; class CTxMemPool; -struct NodeContext; namespace Consensus { struct Params; } +namespace node { +struct NodeContext; + /** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. * Also used by the GUI when broadcasting a completed PSBT. * By default, a transaction with a fee rate higher than this will be rejected @@ -57,5 +59,6 @@ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; * @returns The tx if found, otherwise nullptr */ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); +} // namespace node #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 9b0df96a75..401d4baaeb 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -9,6 +9,7 @@ #include <uint256.h> #include <serialize.h> +namespace node { //! Metadata describing a serialized version of a UTXO set from which an //! assumeutxo CChainState can be constructed. class SnapshotMetadata @@ -32,5 +33,6 @@ public: SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } }; +} // namespace node #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/policy/packages.h b/src/policy/packages.h index d2744f1265..9f274f6b7d 100644 --- a/src/policy/packages.h +++ b/src/policy/packages.h @@ -25,6 +25,7 @@ enum class PackageValidationResult { PCKG_RESULT_UNSET = 0, //!< Initial value. The package has not yet been rejected. PCKG_POLICY, //!< The package itself is invalid (e.g. too many transactions). PCKG_TX, //!< At least one tx is invalid. + PCKG_MEMPOOL_ERROR, //!< Mempool logic error. }; /** A package is an ordered list of transactions. The transactions cannot conflict with (spend the diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 3002e0fe88..6a2781079c 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -41,6 +41,7 @@ #endif // ENABLE_WALLET #include <boost/signals2/connection.hpp> +#include <chrono> #include <memory> #include <QApplication> @@ -76,6 +77,8 @@ Q_DECLARE_METATYPE(CAmount) Q_DECLARE_METATYPE(SynchronizationState) Q_DECLARE_METATYPE(uint256) +using node::NodeContext; + static void RegisterMetaTypes() { // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection @@ -410,10 +413,10 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead connect(paymentServer, &PaymentServer::message, [this](const QString& title, const QString& message, unsigned int style) { window->message(title, message, style); }); - QTimer::singleShot(100, paymentServer, &PaymentServer::uiReady); + QTimer::singleShot(100ms, paymentServer, &PaymentServer::uiReady); } #endif - pollShutdownTimer->start(200); + pollShutdownTimer->start(SHUTDOWN_POLLING_DELAY); } else { Q_EMIT splashFinished(); // Make sure splash screen doesn't stick around during shutdown requestShutdown(); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index d3519edf7f..a28329082a 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -17,6 +17,7 @@ #include <netbase.h> #include <util/system.h> #include <util/threadnames.h> +#include <util/time.h> #include <validation.h> #include <stdint.h> @@ -221,12 +222,12 @@ QString ClientModel::formatClientStartupTime() const QString ClientModel::dataDir() const { - return GUIUtil::boostPathToQString(gArgs.GetDataDirNet()); + return GUIUtil::PathToQString(gArgs.GetDataDirNet()); } QString ClientModel::blocksDir() const { - return GUIUtil::boostPathToQString(gArgs.GetBlocksDirPath()); + return GUIUtil::PathToQString(gArgs.GetBlocksDirPath()); } void ClientModel::updateBanlist() @@ -288,7 +289,7 @@ static void BlockTipChanged(ClientModel* clientmodel, SynchronizationState sync_ const bool throttle = (sync_state != SynchronizationState::POST_INIT && !fHeader) || sync_state == SynchronizationState::INIT_REINDEX; const int64_t now = throttle ? GetTimeMillis() : 0; int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; - if (throttle && now < nLastUpdateNotification + MODEL_UPDATE_DELAY) { + if (throttle && now < nLastUpdateNotification + count_milliseconds(MODEL_UPDATE_DELAY)) { return; } diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 71d1ca0a33..d7a2aaaf19 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -31,6 +31,9 @@ #include <QSettings> #include <QTreeWidget> +using wallet::CCoinControl; +using wallet::MIN_CHANGE; + QList<CAmount> CoinControlDialog::payAmounts; bool CoinControlDialog::fSubtractFeeFromAmount = false; diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 92fae45110..ccceff10e8 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -19,7 +19,9 @@ class PlatformStyle; class WalletModel; +namespace wallet { class CCoinControl; +} // namespace wallet namespace Ui { class CoinControlDialog; @@ -42,11 +44,11 @@ class CoinControlDialog : public QDialog Q_OBJECT public: - explicit CoinControlDialog(CCoinControl& coin_control, WalletModel* model, const PlatformStyle *platformStyle, QWidget *parent = nullptr); + explicit CoinControlDialog(wallet::CCoinControl& coin_control, WalletModel* model, const PlatformStyle *platformStyle, QWidget *parent = nullptr); ~CoinControlDialog(); // static because also called from sendcoinsdialog - static void updateLabels(CCoinControl& m_coin_control, WalletModel*, QDialog*); + static void updateLabels(wallet::CCoinControl& m_coin_control, WalletModel*, QDialog*); static QList<CAmount> payAmounts; static bool fSubtractFeeFromAmount; @@ -56,7 +58,7 @@ protected: private: Ui::CoinControlDialog *ui; - CCoinControl& m_coin_control; + wallet::CCoinControl& m_coin_control; WalletModel *model; int sortColumn; Qt::SortOrder sortOrder; diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 6b3a4630a3..5438811aff 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -252,6 +252,16 @@ </property> </widget> </item> + <item> + <widget class="QCheckBox" name="m_enable_psbt_controls"> + <property name="text"> + <string extracomment="An options window setting to enable PSBT controls.">Enable &PSBT controls</string> + </property> + <property name="toolTip"> + <string extracomment="Tooltip text for options window setting that enables PSBT controls.">Whether to show PSBT controls.</string> + </property> + </widget> + </item> </layout> </widget> </item> diff --git a/src/qt/guiconstants.h b/src/qt/guiconstants.h index 882d2c8f52..1adcd5b6b9 100644 --- a/src/qt/guiconstants.h +++ b/src/qt/guiconstants.h @@ -5,10 +5,16 @@ #ifndef BITCOIN_QT_GUICONSTANTS_H #define BITCOIN_QT_GUICONSTANTS_H +#include <chrono> #include <cstdint> -/* Milliseconds between model updates */ -static const int MODEL_UPDATE_DELAY = 250; +using namespace std::chrono_literals; + +/* A delay between model updates */ +static constexpr auto MODEL_UPDATE_DELAY{250ms}; + +/* A delay between shutdown pollings */ +static constexpr auto SHUTDOWN_POLLING_DELAY{200ms}; /* AskPassphraseDialog -- Maximum passphrase length */ static const int MAX_PASSPHRASE_SIZE = 1024; diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index f9f0ea81ba..c6c8f7b7a6 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -281,7 +281,7 @@ void LoadFont(const QString& file_name) QString getDefaultDataDirectory() { - return boostPathToQString(GetDefaultDataDir()); + return PathToQString(GetDefaultDataDir()); } QString getSaveFileName(QWidget *parent, const QString &caption, const QString &dir, @@ -418,7 +418,7 @@ void openDebugLogfile() /* Open debug.log with the associated application */ if (fs::exists(pathDebug)) - QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathDebug))); + QDesktopServices::openUrl(QUrl::fromLocalFile(PathToQString(pathDebug))); } bool openBitcoinConf() @@ -434,11 +434,11 @@ bool openBitcoinConf() configFile.close(); /* Open bitcoin.conf with the associated application */ - bool res = QDesktopServices::openUrl(QUrl::fromLocalFile(boostPathToQString(pathConfig))); + bool res = QDesktopServices::openUrl(QUrl::fromLocalFile(PathToQString(pathConfig))); #ifdef Q_OS_MAC // Workaround for macOS-specific behavior; see #15409. if (!res) { - res = QProcess::startDetached("/usr/bin/open", QStringList{"-t", boostPathToQString(pathConfig)}); + res = QProcess::startDetached("/usr/bin/open", QStringList{"-t", PathToQString(pathConfig)}); } #endif @@ -652,12 +652,12 @@ void setClipboard(const QString& str) } } -fs::path qstringToBoostPath(const QString &path) +fs::path QStringToPath(const QString &path) { return fs::u8path(path.toStdString()); } -QString boostPathToQString(const fs::path &path) +QString PathToQString(const fs::path &path) { return QString::fromStdString(path.u8string()); } diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index aaed0ea690..9b25b77325 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -209,10 +209,10 @@ namespace GUIUtil bool SetStartOnSystemStartup(bool fAutoStart); /** Convert QString to OS specific boost path through UTF-8 */ - fs::path qstringToBoostPath(const QString &path); + fs::path QStringToPath(const QString &path); /** Convert OS specific boost path to QString through UTF-8 */ - QString boostPathToQString(const fs::path &path); + QString PathToQString(const fs::path &path); /** Convert enum Network to QString */ QString NetworkToQString(Network net); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index c2d6104658..e9a4034e62 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -67,7 +67,7 @@ FreespaceChecker::FreespaceChecker(Intro *_intro) void FreespaceChecker::check() { QString dataDirStr = intro->getPathToCheck(); - fs::path dataDir = GUIUtil::qstringToBoostPath(dataDirStr); + fs::path dataDir = GUIUtil::QStringToPath(dataDirStr); uint64_t freeBytesAvailable = 0; int replyStatus = ST_OK; QString replyMessage = tr("A new data directory will be created."); @@ -216,7 +216,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) /* 2) Allow QSettings to override default dir */ dataDir = settings.value("strDataDir", dataDir).toString(); - if(!fs::exists(GUIUtil::qstringToBoostPath(dataDir)) || gArgs.GetBoolArg("-choosedatadir", DEFAULT_CHOOSE_DATADIR) || settings.value("fReset", false).toBool() || gArgs.GetBoolArg("-resetguisettings", false)) + if(!fs::exists(GUIUtil::QStringToPath(dataDir)) || gArgs.GetBoolArg("-choosedatadir", DEFAULT_CHOOSE_DATADIR) || settings.value("fReset", false).toBool() || gArgs.GetBoolArg("-resetguisettings", false)) { /* Use selectParams here to guarantee Params() can be used by node interface */ try { @@ -240,9 +240,9 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) } dataDir = intro.getDataDirectory(); try { - if (TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir))) { + if (TryCreateDirectories(GUIUtil::QStringToPath(dataDir))) { // If a new data directory has been created, make wallets subdirectory too - TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir) / "wallets"); + TryCreateDirectories(GUIUtil::QStringToPath(dataDir) / "wallets"); } break; } catch (const fs::filesystem_error&) { @@ -263,7 +263,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) * (to be consistent with bitcoind behavior) */ if(dataDir != GUIUtil::getDefaultDataDirectory()) { - gArgs.SoftSetArg("-datadir", fs::PathToString(GUIUtil::qstringToBoostPath(dataDir))); // use OS locale for path setting + gArgs.SoftSetArg("-datadir", fs::PathToString(GUIUtil::QStringToPath(dataDir))); // use OS locale for path setting } return true; } diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index a851f99a3a..c05571677c 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -19,6 +19,8 @@ #include <netbase.h> #include <txdb.h> // for -dbcache defaults +#include <chrono> + #include <QDataWidgetMapper> #include <QDir> #include <QIntValidator> @@ -242,6 +244,7 @@ void OptionsDialog::setMapper() mapper->addMapping(ui->coinControlFeatures, OptionsModel::CoinControlFeatures); mapper->addMapping(ui->subFeeFromAmount, OptionsModel::SubFeeFromAmount); mapper->addMapping(ui->externalSignerPath, OptionsModel::ExternalSignerPath); + mapper->addMapping(ui->m_enable_psbt_controls, OptionsModel::EnablePSBTControls); /* Network */ mapper->addMapping(ui->mapPortUpnp, OptionsModel::MapPortUPnP); @@ -361,7 +364,7 @@ void OptionsDialog::showRestartWarning(bool fPersistent) ui->statusLabel->setText(tr("This change would require a client restart.")); // clear non-persistent status label after 10 seconds // Todo: should perhaps be a class attribute, if we extend the use of statusLabel - QTimer::singleShot(10000, this, &OptionsDialog::clearStatusLabel); + QTimer::singleShot(10s, this, &OptionsDialog::clearStatusLabel); } } diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index fe40c9ef3c..5d9ed5bf23 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -83,6 +83,11 @@ void OptionsModel::Init(bool resetSettings) settings.setValue("fCoinControlFeatures", false); fCoinControlFeatures = settings.value("fCoinControlFeatures", false).toBool(); + if (!settings.contains("enable_psbt_controls")) { + settings.setValue("enable_psbt_controls", false); + } + m_enable_psbt_controls = settings.value("enable_psbt_controls", false).toBool(); + // These are shared with the core or have a command-line parameter // and we want command-line parameters to overwrite the GUI settings. // @@ -204,8 +209,8 @@ static void CopySettings(QSettings& dst, const QSettings& src) /** Back up a QSettings to an ini-formatted file. */ static void BackupSettings(const fs::path& filename, const QSettings& src) { - qInfo() << "Backing up GUI settings to" << GUIUtil::boostPathToQString(filename); - QSettings dst(GUIUtil::boostPathToQString(filename), QSettings::IniFormat); + qInfo() << "Backing up GUI settings to" << GUIUtil::PathToQString(filename); + QSettings dst(GUIUtil::PathToQString(filename), QSettings::IniFormat); dst.clear(); CopySettings(dst, src); } @@ -360,6 +365,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return m_use_embedded_monospaced_font; case CoinControlFeatures: return fCoinControlFeatures; + case EnablePSBTControls: + return settings.value("enable_psbt_controls"); case Prune: return settings.value("bPrune"); case PruneSize: @@ -507,6 +514,10 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in settings.setValue("fCoinControlFeatures", fCoinControlFeatures); Q_EMIT coinControlFeaturesChanged(fCoinControlFeatures); break; + case EnablePSBTControls: + m_enable_psbt_controls = value.toBool(); + settings.setValue("enable_psbt_controls", m_enable_psbt_controls); + break; case Prune: if (settings.value("bPrune") != value) { settings.setValue("bPrune", value); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 059fe2381f..bb9a8c1f8c 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -69,6 +69,7 @@ public: SpendZeroConfChange, // bool Listen, // bool Server, // bool + EnablePSBTControls, // bool OptionIDRowCount, }; @@ -90,6 +91,7 @@ public: bool getUseEmbeddedMonospacedFont() const { return m_use_embedded_monospaced_font; } bool getCoinControlFeatures() const { return fCoinControlFeatures; } bool getSubFeeFromAmount() const { return m_sub_fee_from_amount; } + bool getEnablePSBTControls() const { return m_enable_psbt_controls; } const QString& getOverriddenByCommandLine() { return strOverriddenByCommandLine; } /* Explicit setters */ @@ -115,6 +117,7 @@ private: bool m_use_embedded_monospaced_font; bool fCoinControlFeatures; bool m_sub_fee_from_amount; + bool m_enable_psbt_controls; /* settings that were overridden by command-line */ QString strOverriddenByCommandLine; diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 0cf311a360..cb09035b45 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -51,7 +51,7 @@ static QString ipcServerName() // Append a simple hash of the datadir // Note that gArgs.GetDataDirNet() returns a different path // for -testnet versus main net - QString ddir(GUIUtil::boostPathToQString(gArgs.GetDataDirNet())); + QString ddir(GUIUtil::PathToQString(gArgs.GetDataDirNet())); name.append(QString::number(qHash(ddir))); return name; diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 800c493a05..0962dfe9db 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -17,6 +17,9 @@ #include <iostream> +using node::AnalyzePSBT; +using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::PSBTAnalysis; PSBTOperationsDialog::PSBTOperationsDialog( QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent, GUIUtil::dialog_flags), diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index d76f9fade0..50436a46d8 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -24,17 +24,21 @@ #include <node/ui_interface.h> #include <policy/fees.h> #include <txmempool.h> +#include <validation.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> #include <wallet/wallet.h> -#include <validation.h> +#include <chrono> #include <QFontMetrics> #include <QScrollBar> #include <QSettings> #include <QTextDocument> +using wallet::CCoinControl; +using wallet::DEFAULT_PAY_TX_FEE; + static constexpr std::array confTargets{2, 4, 6, 12, 24, 48, 144, 504, 1008}; int getConfTargetForIndex(int index) { if (index+1 > static_cast<int>(confTargets.size())) { @@ -324,16 +328,22 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa formatted.append(recipientElement); } - if (model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner()) { - question_string.append(tr("Do you want to draft this transaction?")); - } else { - question_string.append(tr("Are you sure you want to send?")); - } - + /*: Message displayed when attempting to create a transaction. Cautionary text to prompt the user to verify + that the displayed transaction details represent the transaction the user intends to create. */ + question_string.append(tr("Do you want to create this transaction?")); question_string.append("<br /><span style='font-size:10pt;'>"); if (model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner()) { + /*: Text to inform a user attempting to create a transaction of their current options. At this stage, + a user can only create a PSBT. This string is displayed when private keys are disabled and an external + signer is not available. */ question_string.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can save or copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + } else if (model->getOptionsModel()->getEnablePSBTControls()) { + /*: Text to inform a user attempting to create a transaction of their current options. At this stage, + a user can send their transaction or create a PSBT. This string is displayed when both private keys + and PSBT controls are enabled. */ + question_string.append(tr("Please, review your transaction. You can create and send this transaction or create a Partially Signed Bitcoin Transaction (PSBT), which you can save or copy and then sign with, e.g., an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); } else { + /*: Text to prompt a user to review the details of the transaction they are attempting to send. */ question_string.append(tr("Please, review your transaction.")); } question_string.append("</span>%1"); @@ -397,21 +407,20 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) if (!PrepareSendText(question_string, informative_text, detailed_text)) return; assert(m_current_transaction); - const QString confirmation = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner() ? tr("Confirm transaction proposal") : tr("Confirm send coins"); - const QString confirmButtonText = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner() ? tr("Create Unsigned") : tr("Sign and send"); - auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); + const QString confirmation = tr("Confirm send coins"); + auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, !model->wallet().privateKeysDisabled(), model->getOptionsModel()->getEnablePSBTControls(), this); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); - if(retval != QMessageBox::Yes) + if(retval != QMessageBox::Yes && retval != QMessageBox::Save) { fNewRecipientAllowed = true; return; } bool send_failure = false; - if (model->wallet().privateKeysDisabled()) { + if (retval == QMessageBox::Save) { CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; PartiallySignedTransaction psbtx(mtx); bool complete = false; @@ -512,6 +521,7 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) assert(false); } // msgBox.exec() } else { + assert(!model->wallet().privateKeysDisabled()); // now send the prepared transaction WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); // process sendStatus and on error generate message shown to user @@ -1031,8 +1041,8 @@ void SendCoinsDialog::coinControlUpdateLabels() } } -SendConfirmationDialog::SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text, const QString& detailed_text, int _secDelay, const QString& _confirmButtonText, QWidget* parent) - : QMessageBox(parent), secDelay(_secDelay), confirmButtonText(_confirmButtonText) +SendConfirmationDialog::SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text, const QString& detailed_text, int _secDelay, bool enable_send, bool always_show_unsigned, QWidget* parent) + : QMessageBox(parent), secDelay(_secDelay), m_enable_send(enable_send) { setIcon(QMessageBox::Question); setWindowTitle(title); // On macOS, the window title is ignored (as required by the macOS Guidelines). @@ -1040,26 +1050,28 @@ SendConfirmationDialog::SendConfirmationDialog(const QString& title, const QStri setInformativeText(informative_text); setDetailedText(detailed_text); setStandardButtons(QMessageBox::Yes | QMessageBox::Cancel); + if (always_show_unsigned || !enable_send) addButton(QMessageBox::Save); setDefaultButton(QMessageBox::Cancel); yesButton = button(QMessageBox::Yes); if (confirmButtonText.isEmpty()) { confirmButtonText = yesButton->text(); } - updateYesButton(); + m_psbt_button = button(QMessageBox::Save); + updateButtons(); connect(&countDownTimer, &QTimer::timeout, this, &SendConfirmationDialog::countDown); } int SendConfirmationDialog::exec() { - updateYesButton(); - countDownTimer.start(1000); + updateButtons(); + countDownTimer.start(1s); return QMessageBox::exec(); } void SendConfirmationDialog::countDown() { secDelay--; - updateYesButton(); + updateButtons(); if(secDelay <= 0) { @@ -1067,16 +1079,24 @@ void SendConfirmationDialog::countDown() } } -void SendConfirmationDialog::updateYesButton() +void SendConfirmationDialog::updateButtons() { if(secDelay > 0) { yesButton->setEnabled(false); - yesButton->setText(confirmButtonText + " (" + QString::number(secDelay) + ")"); + yesButton->setText(confirmButtonText + (m_enable_send ? (" (" + QString::number(secDelay) + ")") : QString(""))); + if (m_psbt_button) { + m_psbt_button->setEnabled(false); + m_psbt_button->setText(m_psbt_button_text + " (" + QString::number(secDelay) + ")"); + } } else { - yesButton->setEnabled(true); + yesButton->setEnabled(m_enable_send); yesButton->setText(confirmButtonText); + if (m_psbt_button) { + m_psbt_button->setEnabled(true); + m_psbt_button->setText(m_psbt_button_text); + } } } diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 4e43697f78..4a16702756 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -12,12 +12,14 @@ #include <QString> #include <QTimer> -class CCoinControl; class ClientModel; class PlatformStyle; class SendCoinsEntry; class SendCoinsRecipient; enum class SynchronizationState; +namespace wallet { +class CCoinControl; +} // namespace wallet namespace Ui { class SendCoinsDialog; @@ -62,7 +64,7 @@ private: Ui::SendCoinsDialog *ui; ClientModel *clientModel; WalletModel *model; - std::unique_ptr<CCoinControl> m_coin_control; + std::unique_ptr<wallet::CCoinControl> m_coin_control; std::unique_ptr<WalletModelTransaction> m_current_transaction; bool fNewRecipientAllowed; bool fFeeMinimized; @@ -114,18 +116,21 @@ class SendConfirmationDialog : public QMessageBox Q_OBJECT public: - SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, const QString& confirmText = "", QWidget* parent = nullptr); + SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, bool enable_send = true, bool always_show_unsigned = true, QWidget* parent = nullptr); int exec() override; private Q_SLOTS: void countDown(); - void updateYesButton(); + void updateButtons(); private: QAbstractButton *yesButton; + QAbstractButton *m_psbt_button; QTimer countDownTimer; int secDelay; - QString confirmButtonText; + QString confirmButtonText{tr("Send")}; + bool m_enable_send; + QString m_psbt_button_text{tr("Create Unsigned")}; }; #endif // BITCOIN_QT_SENDCOINSDIALOG_H diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 9eabecbfc9..66637a5dcf 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -20,10 +20,19 @@ #include <wallet/wallet.h> #include <walletinitinterface.h> +#include <chrono> + #include <QApplication> #include <QTimer> #include <QMessageBox> +using wallet::AddWallet; +using wallet::CWallet; +using wallet::CreateMockWalletDatabase; +using wallet::RemoveWallet; +using wallet::WALLET_FLAG_DESCRIPTORS; +using wallet::WalletContext; + namespace { @@ -40,7 +49,7 @@ void EditAddressAndSubmit( dialog->findChild<QLineEdit*>("labelEdit")->setText(label); dialog->findChild<QValidatedLineEdit*>("addressEdit")->setText(address); - ConfirmMessage(&warning_text, 5); + ConfirmMessage(&warning_text, 5ms); dialog->accept(); QCOMPARE(warning_text, expected_msg); } diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 5f7b5e429d..10b7e2ffe7 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -22,6 +22,7 @@ #include <QApplication> #include <QObject> #include <QTest> +#include <functional> #if defined(QT_STATICPLUGIN) #include <QtPlugin> @@ -39,8 +40,12 @@ Q_IMPORT_PLUGIN(QAndroidPlatformIntegrationPlugin) #endif #endif +using node::NodeContext; + const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; +const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; + // This is all you need to run all the tests int main(int argc, char* argv[]) { diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index 987d921f03..635dbcd1c5 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <chrono> + #include <QApplication> #include <QMessageBox> #include <QPushButton> @@ -9,7 +11,7 @@ #include <QTimer> #include <QWidget> -void ConfirmMessage(QString* text, int msec) +void ConfirmMessage(QString* text, std::chrono::milliseconds msec) { QTimer::singleShot(msec, [text]() { for (QWidget* widget : QApplication::topLevelWidgets()) { diff --git a/src/qt/test/util.h b/src/qt/test/util.h index df5931a032..f50a6b6c61 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -5,7 +5,11 @@ #ifndef BITCOIN_QT_TEST_UTIL_H #define BITCOIN_QT_TEST_UTIL_H -#include <QString> +#include <chrono> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE /** * Press "Ok" button in message box dialog. @@ -13,6 +17,6 @@ * @param text - Optionally store dialog text. * @param msec - Number of milliseconds to pause before triggering the callback. */ -void ConfirmMessage(QString* text = nullptr, int msec = 0); +void ConfirmMessage(QString* text, std::chrono::milliseconds msec); #endif // BITCOIN_QT_TEST_UTIL_H diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 147a8a076b..6ab534764b 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -26,6 +26,7 @@ #include <qt/recentrequeststablemodel.h> #include <qt/receiverequestdialog.h> +#include <chrono> #include <memory> #include <QAbstractButton> @@ -39,6 +40,15 @@ #include <QListView> #include <QDialogButtonBox> +using wallet::AddWallet; +using wallet::CWallet; +using wallet::CreateMockWalletDatabase; +using wallet::RemoveWallet; +using wallet::WALLET_FLAG_DESCRIPTORS; +using wallet::WalletContext; +using wallet::WalletDescriptor; +using wallet::WalletRescanReserver; + namespace { //! Press "Yes" or "Cancel" buttons in modal send confirmation dialog. @@ -112,7 +122,7 @@ void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, st if (expectError.empty()) { ConfirmSend(&text, cancel); } else { - ConfirmMessage(&text); + ConfirmMessage(&text, 0ms); } action->trigger(); QVERIFY(text.indexOf(QString::fromStdString(expectError)) != -1); diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 260e9eeb33..0504639cde 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -28,6 +28,11 @@ #include <QLatin1String> +using wallet::ISMINE_ALL; +using wallet::ISMINE_SPENDABLE; +using wallet::ISMINE_WATCH_ONLY; +using wallet::isminetype; + QString TransactionDesc::FormatTxStatus(const interfaces::WalletTx& wtx, const interfaces::WalletTxStatus& status, bool inMempool, int numBlocks) { if (!status.is_final) diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 44714de146..5386569973 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -13,6 +13,10 @@ #include <QDateTime> +using wallet::ISMINE_SPENDABLE; +using wallet::ISMINE_WATCH_ONLY; +using wallet::isminetype; + /* Return positive answer if transaction should be shown in list. */ bool TransactionRecord::showTransaction() diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 6f6e40fcf7..1ab1333b72 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -19,6 +19,7 @@ #include <node/ui_interface.h> +#include <chrono> #include <optional> #include <QApplication> @@ -114,8 +115,8 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa amountWidget->setValidator(amountValidator); hlayout->addWidget(amountWidget); - // Delay before filtering transactions in ms - static const int input_filter_delay = 200; + // Delay before filtering transactions + static constexpr auto input_filter_delay{200ms}; QTimer* amount_typing_delay = new QTimer(this); amount_typing_delay->setSingleShot(true); diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index fa561e05db..b025bb367c 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -20,6 +20,7 @@ #include <wallet/wallet.h> #include <algorithm> +#include <chrono> #include <QApplication> #include <QMessageBox> @@ -28,6 +29,11 @@ #include <QTimer> #include <QWindow> +using wallet::WALLET_FLAG_BLANK_WALLET; +using wallet::WALLET_FLAG_DESCRIPTORS; +using wallet::WALLET_FLAG_DISABLE_PRIVATE_KEYS; +using wallet::WALLET_FLAG_EXTERNAL_SIGNER; + WalletController::WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent) : QObject(parent) , m_activity_thread(new QThread(this)) @@ -254,12 +260,12 @@ void CreateWalletActivity::createWallet() flags |= WALLET_FLAG_EXTERNAL_SIGNER; } - QTimer::singleShot(500, worker(), [this, name, flags] { + QTimer::singleShot(500ms, worker(), [this, name, flags] { std::unique_ptr<interfaces::Wallet> wallet = node().walletLoader().createWallet(name, m_passphrase, flags, m_error_message, m_warning_message); if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); - QTimer::singleShot(500, this, &CreateWalletActivity::finish); + QTimer::singleShot(500ms, this, &CreateWalletActivity::finish); }); } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index cfc255c843..5ee32e79d5 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -36,6 +36,9 @@ #include <QSet> #include <QTimer> +using wallet::CCoinControl; +using wallet::CRecipient; +using wallet::DEFAULT_DISABLE_WALLET; WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : QObject(parent), @@ -480,10 +483,9 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return false; } - const bool create_psbt = m_wallet->privateKeysDisabled(); - // allow a user based fee verification - QString questionString = create_psbt ? tr("Do you want to draft a transaction with fee increase?") : tr("Do you want to increase the fee?"); + /*: Asks a user if they would like to manually increase the fee of a transaction that has already been created. */ + QString questionString = tr("Do you want to increase the fee?"); questionString.append("<br />"); questionString.append("<table style=\"text-align: left;\">"); questionString.append("<tr><td>"); @@ -506,13 +508,13 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); } - auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString); + auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, !m_wallet->privateKeysDisabled(), getOptionsModel()->getEnablePSBTControls(), nullptr); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); // cancel sign&broadcast if user doesn't want to bump the fee - if (retval != QMessageBox::Yes) { + if (retval != QMessageBox::Yes && retval != QMessageBox::Save) { return false; } @@ -523,7 +525,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) } // Short-circuit if we are returning a bumped transaction PSBT to clipboard - if (create_psbt) { + if (retval == QMessageBox::Save) { PartiallySignedTransaction psbtx(mtx); bool complete = false; const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); @@ -539,6 +541,8 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } + assert(!m_wallet->privateKeysDisabled()); + // sign bumped transaction if (!m_wallet->signBumpTransaction(mtx)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't sign transaction.")); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 3097a4f78f..ad1239ccdc 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -32,16 +32,17 @@ class SendCoinsRecipient; class TransactionTableModel; class WalletModelTransaction; -class CCoinControl; class CKeyID; class COutPoint; -class COutput; class CPubKey; class uint256; namespace interfaces { class Node; } // namespace interfaces +namespace wallet { +class CCoinControl; +} // namespace wallet QT_BEGIN_NAMESPACE class QTimer; @@ -99,7 +100,7 @@ public: }; // prepare transaction for getting txfee before sending coins - SendCoinsReturn prepareTransaction(WalletModelTransaction &transaction, const CCoinControl& coinControl); + SendCoinsReturn prepareTransaction(WalletModelTransaction &transaction, const wallet::CCoinControl& coinControl); // Send coins to a list of recipients SendCoinsReturn sendCoins(WalletModelTransaction &transaction); diff --git a/src/random.cpp b/src/random.cpp index 6eb06c5d47..5dae80fe31 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -19,6 +19,7 @@ #include <sync.h> // for Mutex #include <util/time.h> // for GetTimeMicros() +#include <cmath> #include <stdlib.h> #include <thread> @@ -714,3 +715,9 @@ void RandomInit() ReportHardwareRand(); } + +std::chrono::microseconds GetExponentialRand(std::chrono::microseconds now, std::chrono::seconds average_interval) +{ + double unscaled = -std::log1p(GetRand(uint64_t{1} << 48) * -0.0000000000000035527136788 /* -1/2^48 */); + return now + std::chrono::duration_cast<std::chrono::microseconds>(unscaled * average_interval + 0.5us); +} diff --git a/src/random.h b/src/random.h index 0c6dc24983..5174c553fb 100644 --- a/src/random.h +++ b/src/random.h @@ -10,7 +10,7 @@ #include <crypto/common.h> #include <uint256.h> -#include <chrono> // For std::chrono::microseconds +#include <chrono> #include <cstdint> #include <limits> @@ -82,6 +82,18 @@ D GetRandomDuration(typename std::common_type<D>::type max) noexcept }; constexpr auto GetRandMicros = GetRandomDuration<std::chrono::microseconds>; constexpr auto GetRandMillis = GetRandomDuration<std::chrono::milliseconds>; + +/** + * Return a timestamp in the future sampled from an exponential distribution + * (https://en.wikipedia.org/wiki/Exponential_distribution). This distribution + * is memoryless and should be used for repeated network events (e.g. sending a + * certain type of message) to minimize leaking information to observers. + * + * The probability of an event occuring before time x is 1 - e^-(x/a) where a + * is the average interval between events. + * */ +std::chrono::microseconds GetExponentialRand(std::chrono::microseconds now, std::chrono::seconds average_interval); + int GetRandInt(int nMax) noexcept; uint256 GetRandHash() noexcept; diff --git a/src/rest.cpp b/src/rest.cpp index addce6b0ea..063872b47a 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -32,6 +32,11 @@ #include <univalue.h> +using node::GetTransaction; +using node::IsBlockPruned; +using node::NodeContext; +using node::ReadBlockFromDisk; + static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 20a5eea173..ccc859619d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,10 +19,10 @@ #include <hash.h> #include <index/blockfilterindex.h> #include <index/coinstatsindex.h> +#include <logging/timer.h> #include <net.h> #include <net_processing.h> #include <node/blockstorage.h> -#include <logging/timer.h> #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> @@ -56,6 +56,16 @@ #include <memory> #include <mutex> +using node::BlockManager; +using node::CCoinsStats; +using node::CoinStatsHashType; +using node::GetUTXOStats; +using node::IsBlockPruned; +using node::NodeContext; +using node::ReadBlockFromDisk; +using node::SnapshotMetadata; +using node::UndoReadFromDisk; + struct CUpdatedBlock { uint256 hash; @@ -1112,7 +1122,7 @@ static RPCHelpMan pruneblockchain() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (!fPruneMode) + if (!node::fPruneMode) throw JSONRPCError(RPC_MISC_ERROR, "Cannot prune blocks because node is not in prune mode."); ChainstateManager& chainman = EnsureAnyChainman(request.context); @@ -1274,9 +1284,10 @@ static RPCHelpMan gettxoutsetinfo() ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); } if (hash_type == CoinStatsHashType::MUHASH) { - ret.pushKV("muhash", stats.hashSerialized.GetHex()); + ret.pushKV("muhash", stats.hashSerialized.GetHex()); } - ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); + CHECK_NONFATAL(stats.total_amount.has_value()); + ret.pushKV("total_amount", ValueFromAmount(stats.total_amount.value())); if (!stats.index_used) { ret.pushKV("transactions", static_cast<int64_t>(stats.nTransactions)); ret.pushKV("disk_size", stats.nDiskSize); @@ -1563,9 +1574,9 @@ RPCHelpMan getblockchaininfo() obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); obj.pushKV("chainwork", tip->nChainWork.GetHex()); - obj.pushKV("size_on_disk", CalculateCurrentUsage()); - obj.pushKV("pruned", fPruneMode); - if (fPruneMode) { + obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); + obj.pushKV("pruned", node::fPruneMode); + if (node::fPruneMode) { const CBlockIndex* block = tip; CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { @@ -1578,7 +1589,7 @@ RPCHelpMan getblockchaininfo() bool automatic_pruning{args.GetIntArg("-prune", 0) != 1}; obj.pushKV("automatic_pruning", automatic_pruning); if (automatic_pruning) { - obj.pushKV("prune_target_size", nPruneTarget); + obj.pushKV("prune_target_size", node::nPruneTarget); } } @@ -1742,7 +1753,7 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::NUM, "size", "Current tx count"}, {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, - {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritizetransaction"}, + {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritisetransaction"}, {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 2176b5997e..1f51d7c1ad 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -22,7 +22,9 @@ class CBlockIndex; class CChainState; class CTxMemPool; class UniValue; +namespace node { struct NodeContext; +} // namespace node static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -57,7 +59,7 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], * @return a UniValue map containing metadata about the snapshot. */ UniValue CreateUTXOSnapshot( - NodeContext& node, + node::NodeContext& node, CChainState& chainstate, CAutoFile& afile, const fs::path& path, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 3042f1657b..0554367672 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -41,6 +41,13 @@ #include <memory> #include <stdint.h> +using node::BlockAssembler; +using node::CBlockTemplate; +using node::IncrementExtraNonce; +using node::NodeContext; +using node::RegenerateCommitments; +using node::UpdateTime; + /** * Return average network hashes per second based on the last 'lookup' blocks, * or from the last difficulty change if 'lookup' is nonpositive. diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 0e33bd6f28..8d574e0359 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -35,6 +35,8 @@ #include <univalue.h> +using node::NodeContext; + static RPCHelpMan validateaddress() { return RPCHelpMan{ diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 71c5ceadd6..3d7c00edfc 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -32,6 +32,8 @@ #include <univalue.h> +using node::NodeContext; + const std::vector<std::string> CONNECTION_TYPE_DOC{ "outbound-full-relay (default automatic connections)", "block-relay-only (does not relay transactions or addresses)", @@ -646,7 +648,7 @@ static RPCHelpMan getnetworkinfo() obj.pushKV("incrementalfee", ValueFromAmount(::incrementalRelayFee.GetFeePerK())); UniValue localAddresses(UniValue::VARR); { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); for (const std::pair<const CNetAddr, LocalServiceInfo> &item : mapLocalHost) { UniValue rec(UniValue::VOBJ); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 3c9e0625e6..f227fde0f7 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -45,6 +45,15 @@ #include <univalue.h> +using node::AnalyzePSBT; +using node::BroadcastTransaction; +using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::FindCoins; +using node::GetTransaction; +using node::NodeContext; +using node::PSBTAnalysis; +using node::ReadBlockFromDisk; + static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, CChainState& active_chainstate) { // Call into TxToUniv() in bitcoin-common to decode the transaction hex. @@ -1066,8 +1075,9 @@ static RPCHelpMan testmempoolaccept() static RPCHelpMan decodepsbt() { - return RPCHelpMan{"decodepsbt", - "\nReturn a JSON object representing the serialized, base64-encoded partially signed Bitcoin transaction.\n", + return RPCHelpMan{ + "decodepsbt", + "Return a JSON object representing the serialized, base64-encoded partially signed Bitcoin transaction.", { {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The PSBT base64 string"}, }, @@ -1176,7 +1186,7 @@ static RPCHelpMan decodepsbt() { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, - {RPCResult::Type::ARR, "proprietary", "The input proprietary map", + {RPCResult::Type::ARR, "proprietary", /*optional=*/true, "The input proprietary map", { {RPCResult::Type::OBJ, "", "", { @@ -1217,7 +1227,7 @@ static RPCHelpMan decodepsbt() { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, - {RPCResult::Type::ARR, "proprietary", "The output proprietary map", + {RPCResult::Type::ARR, "proprietary", /*optional=*/true, "The output proprietary map", { {RPCResult::Type::OBJ, "", "", { diff --git a/src/rpc/server_util.cpp b/src/rpc/server_util.cpp index 2978051574..6a1b41f066 100644 --- a/src/rpc/server_util.cpp +++ b/src/rpc/server_util.cpp @@ -15,6 +15,8 @@ #include <any> +using node::NodeContext; + NodeContext& EnsureAnyNodeContext(const std::any& context) { auto node_context = util::AnyPtr<NodeContext>(context); diff --git a/src/rpc/server_util.h b/src/rpc/server_util.h index 3b0df2d651..2cc710a803 100644 --- a/src/rpc/server_util.h +++ b/src/rpc/server_util.h @@ -13,18 +13,20 @@ class CConnman; class CTxMemPool; class ChainstateManager; class PeerManager; +namespace node { struct NodeContext; +} // namespace node -NodeContext& EnsureAnyNodeContext(const std::any& context); -CTxMemPool& EnsureMemPool(const NodeContext& node); +node::NodeContext& EnsureAnyNodeContext(const std::any& context); +CTxMemPool& EnsureMemPool(const node::NodeContext& node); CTxMemPool& EnsureAnyMemPool(const std::any& context); -ArgsManager& EnsureArgsman(const NodeContext& node); +ArgsManager& EnsureArgsman(const node::NodeContext& node); ArgsManager& EnsureAnyArgsman(const std::any& context); -ChainstateManager& EnsureChainman(const NodeContext& node); +ChainstateManager& EnsureChainman(const node::NodeContext& node); ChainstateManager& EnsureAnyChainman(const std::any& context); -CBlockPolicyEstimator& EnsureFeeEstimator(const NodeContext& node); +CBlockPolicyEstimator& EnsureFeeEstimator(const node::NodeContext& node); CBlockPolicyEstimator& EnsureAnyFeeEstimator(const std::any& context); -CConnman& EnsureConnman(const NodeContext& node); -PeerManager& EnsurePeerman(const NodeContext& node); +CConnman& EnsureConnman(const node::NodeContext& node); +PeerManager& EnsurePeerman(const node::NodeContext& node); #endif // BITCOIN_RPC_SERVER_UTIL_H diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 378f866e09..0b2ad3c553 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -136,7 +136,7 @@ bool CScheduler::AreThreadsServicingQueue() const void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() { { - LOCK(m_cs_callbacks_pending); + LOCK(m_callbacks_mutex); // Try to avoid scheduling too many copies here, but if we // accidentally have two ProcessQueue's scheduled at once its // not a big deal. @@ -150,7 +150,7 @@ void SingleThreadedSchedulerClient::ProcessQueue() { std::function<void()> callback; { - LOCK(m_cs_callbacks_pending); + LOCK(m_callbacks_mutex); if (m_are_callbacks_running) return; if (m_callbacks_pending.empty()) return; m_are_callbacks_running = true; @@ -167,7 +167,7 @@ void SingleThreadedSchedulerClient::ProcessQueue() ~RAIICallbacksRunning() { { - LOCK(instance->m_cs_callbacks_pending); + LOCK(instance->m_callbacks_mutex); instance->m_are_callbacks_running = false; } instance->MaybeScheduleProcessQueue(); @@ -182,7 +182,7 @@ void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void()> func assert(m_pscheduler); { - LOCK(m_cs_callbacks_pending); + LOCK(m_callbacks_mutex); m_callbacks_pending.emplace_back(std::move(func)); } MaybeScheduleProcessQueue(); @@ -194,13 +194,13 @@ void SingleThreadedSchedulerClient::EmptyQueue() bool should_continue = true; while (should_continue) { ProcessQueue(); - LOCK(m_cs_callbacks_pending); + LOCK(m_callbacks_mutex); should_continue = !m_callbacks_pending.empty(); } } size_t SingleThreadedSchedulerClient::CallbacksPending() { - LOCK(m_cs_callbacks_pending); + LOCK(m_callbacks_mutex); return m_callbacks_pending.size(); } diff --git a/src/scheduler.h b/src/scheduler.h index 5366a5989c..bb0abfbf7a 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -119,9 +119,9 @@ class SingleThreadedSchedulerClient private: CScheduler* m_pscheduler; - RecursiveMutex m_cs_callbacks_pending; - std::list<std::function<void()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); - bool m_are_callbacks_running GUARDED_BY(m_cs_callbacks_pending) = false; + Mutex m_callbacks_mutex; + std::list<std::function<void()>> m_callbacks_pending GUARDED_BY(m_callbacks_mutex); + bool m_are_callbacks_running GUARDED_BY(m_callbacks_mutex) = false; void MaybeScheduleProcessQueue(); void ProcessQueue(); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 95ffe40a74..07b44971b7 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1500,7 +1500,7 @@ static bool HandleMissingData(MissingDataBehavior mdb) } template<typename T> -bool SignatureHashSchnorr(uint256& hash_out, const ScriptExecutionData& execdata, const T& tx_to, uint32_t in_pos, uint8_t hash_type, SigVersion sigversion, const PrecomputedTransactionData& cache, MissingDataBehavior mdb) +bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, const T& tx_to, uint32_t in_pos, uint8_t hash_type, SigVersion sigversion, const PrecomputedTransactionData& cache, MissingDataBehavior mdb) { uint8_t ext_flag, key_version; switch (sigversion) { @@ -1568,9 +1568,12 @@ bool SignatureHashSchnorr(uint256& hash_out, const ScriptExecutionData& execdata // Data about the output (if only one). if (output_type == SIGHASH_SINGLE) { if (in_pos >= tx_to.vout.size()) return false; - CHashWriter sha_single_output(SER_GETHASH, 0); - sha_single_output << tx_to.vout[in_pos]; - ss << sha_single_output.GetSHA256(); + if (!execdata.m_output_hash) { + CHashWriter sha_single_output(SER_GETHASH, 0); + sha_single_output << tx_to.vout[in_pos]; + execdata.m_output_hash = sha_single_output.GetSHA256(); + } + ss << execdata.m_output_hash.value(); } // Additional data for BIP 342 signatures @@ -1692,7 +1695,7 @@ bool GenericTransactionSignatureChecker<T>::CheckECDSASignature(const std::vecto } template <class T> -bool GenericTransactionSignatureChecker<T>::CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey_in, SigVersion sigversion, const ScriptExecutionData& execdata, ScriptError* serror) const +bool GenericTransactionSignatureChecker<T>::CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey_in, SigVersion sigversion, ScriptExecutionData& execdata, ScriptError* serror) const { assert(sigversion == SigVersion::TAPROOT || sigversion == SigVersion::TAPSCRIPT); // Schnorr signatures have 32-byte public keys. The caller is responsible for enforcing this. diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 2a28f1a2d3..cf1953ad22 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -11,6 +11,7 @@ #include <span.h> #include <primitives/transaction.h> +#include <optional> #include <vector> #include <stdint.h> @@ -215,6 +216,9 @@ struct ScriptExecutionData bool m_validation_weight_left_init = false; //! How much validation weight is left (decremented for every successful non-empty signature check). int64_t m_validation_weight_left; + + //! The hash of the corresponding output + std::optional<uint256> m_output_hash; }; /** Signature hash sizes */ @@ -244,7 +248,7 @@ public: return false; } - virtual bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, const ScriptExecutionData& execdata, ScriptError* serror = nullptr) const + virtual bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, ScriptExecutionData& execdata, ScriptError* serror = nullptr) const { return false; } @@ -272,7 +276,7 @@ enum class MissingDataBehavior }; template<typename T> -bool SignatureHashSchnorr(uint256& hash_out, const ScriptExecutionData& execdata, const T& tx_to, uint32_t in_pos, uint8_t hash_type, SigVersion sigversion, const PrecomputedTransactionData& cache, MissingDataBehavior mdb); +bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, const T& tx_to, uint32_t in_pos, uint8_t hash_type, SigVersion sigversion, const PrecomputedTransactionData& cache, MissingDataBehavior mdb); template <class T> class GenericTransactionSignatureChecker : public BaseSignatureChecker @@ -292,7 +296,7 @@ public: GenericTransactionSignatureChecker(const T* txToIn, unsigned int nInIn, const CAmount& amountIn, MissingDataBehavior mdb) : txTo(txToIn), m_mdb(mdb), nIn(nInIn), amount(amountIn), txdata(nullptr) {} GenericTransactionSignatureChecker(const T* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData& txdataIn, MissingDataBehavior mdb) : txTo(txToIn), m_mdb(mdb), nIn(nInIn), amount(amountIn), txdata(&txdataIn) {} bool CheckECDSASignature(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const override; - bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, const ScriptExecutionData& execdata, ScriptError* serror = nullptr) const override; + bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, ScriptExecutionData& execdata, ScriptError* serror = nullptr) const override; bool CheckLockTime(const CScriptNum& nLockTime) const override; bool CheckSequence(const CScriptNum& nSequence) const override; }; @@ -313,7 +317,7 @@ public: return m_checker.CheckECDSASignature(scriptSig, vchPubKey, scriptCode, sigversion); } - bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, const ScriptExecutionData& execdata, ScriptError* serror = nullptr) const override + bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, ScriptExecutionData& execdata, ScriptError* serror = nullptr) const override { return m_checker.CheckSchnorrSignature(sig, pubkey, sigversion, execdata, serror); } diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 8e08448480..371a937bc8 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -542,7 +542,7 @@ class DummySignatureChecker final : public BaseSignatureChecker public: DummySignatureChecker() {} bool CheckECDSASignature(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const override { return true; } - bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, const ScriptExecutionData& execdata, ScriptError* serror) const override { return true; } + bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, ScriptExecutionData& execdata, ScriptError* serror) const override { return true; } }; const DummySignatureChecker DUMMY_CHECKER; diff --git a/src/test/README.md b/src/test/README.md index d03411c3ed..90d0e7102d 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -33,19 +33,31 @@ the `src/qt/test/test_main.cpp` file. ### Running individual tests -`test_bitcoin` has some built-in command-line arguments; for -example, to run just the `getarg_tests` verbosely: +`test_bitcoin` accepts the command line arguments from the boost framework. +For example, to run just the `getarg_tests` suite of tests: - test_bitcoin --log_level=all --run_test=getarg_tests -- DEBUG_LOG_OUT +```bash +test_bitcoin --log_level=all --run_test=getarg_tests +``` `log_level` controls the verbosity of the test framework, which logs when a -test case is entered, for example. The `DEBUG_LOG_OUT` after the two dashes -redirects the debug log, which would normally go to a file in the test datadir +test case is entered, for example. `test_bitcoin` also accepts the command +line arguments accepted by `bitcoind`. Use `--` to separate both types of +arguments: + +```bash +test_bitcoin --log_level=all --run_test=getarg_tests -- -printtoconsole=1 +``` + +The `-printtoconsole=1` after the two dashes redirects the debug log, which +would normally go to a file in the test datadir (`BasicTestingSetup::m_path_root`), to the standard terminal output. ... or to run just the doubledash test: - test_bitcoin --run_test=getarg_tests/doubledash +```bash +test_bitcoin --run_test=getarg_tests/doubledash +``` Run `test_bitcoin --help` for the full list. @@ -68,7 +80,7 @@ on failure. For running individual tests verbosely, refer to the section To write to logs from unit tests you need to use specific message methods provided by Boost. The simplest is `BOOST_TEST_MESSAGE`. -For debugging you can launch the `test_bitcoin` executable with `gdb`or `lldb` and +For debugging you can launch the `test_bitcoin` executable with `gdb` or `lldb` and start debugging, just like you would with any other program: ```bash @@ -95,7 +107,7 @@ Running the tests and hitting a segmentation fault should now produce a file cal `/proc/sys/kernel/core_pattern`). You can then explore the core dump using -``` bash +```bash gdb src/test/test_bitcoin core (gbd) bt # produce a backtrace for where a segfault occurred diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index fd365e703f..efc30b6822 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -21,59 +21,15 @@ #include <string> using namespace std::literals; +using node::NodeContext; -class AddrManTest : public AddrMan -{ -public: - explicit AddrManTest(std::vector<bool> asmap = std::vector<bool>()) - : AddrMan(asmap, /*deterministic=*/true, /*consistency_check_ratio=*/100) - {} - - AddrInfo* Find(const CService& addr) - { - LOCK(m_impl->cs); - return m_impl->Find(addr); - } - - AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) - { - LOCK(m_impl->cs); - return m_impl->Create(addr, addrSource, pnId); - } - - void Delete(int nId) - { - LOCK(m_impl->cs); - m_impl->Delete(nId); - } - - // Used to test deserialization - std::pair<int, int> GetBucketAndEntry(const CAddress& addr) - { - LOCK(m_impl->cs); - int nId = m_impl->mapAddr[addr]; - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { - for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { - if (nId == m_impl->vvNew[bucket][entry]) { - return std::pair<int, int>(bucket, entry); - } - } - } - return std::pair<int, int>(-1, -1); - } - - // Simulates connection failure so that we can test eviction of offline nodes - void SimConnFail(const CService& addr) - { - int64_t nLastSuccess = 1; - // Set last good connection in the deep past. - Good(addr, nLastSuccess); +static const std::vector<bool> EMPTY_ASMAP; +static const bool DETERMINISTIC{true}; - bool count_failure = false; - int64_t nLastTry = GetAdjustedTime() - 61; - Attempt(addr, count_failure, nLastTry); - } -}; +static int32_t GetCheckRatio(const NodeContext& node_ctx) +{ + return std::clamp<int32_t>(node_ctx.args->GetIntArg("-checkaddrman", 100), 0, 1000000); +} static CNetAddr ResolveIP(const std::string& ip) { @@ -102,12 +58,11 @@ static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) return result; } - BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) { - auto addrman = std::make_unique<AddrManTest>(); + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -141,7 +96,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_CHECK(addrman->size() >= 1); // Test: reset addrman and test AddrMan::Add multiple addresses works as expected - addrman = std::make_unique<AddrManTest>(); + addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); std::vector<CAddress> vAddr; vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); @@ -151,58 +106,58 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_AUTO_TEST_CASE(addrman_ports) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK_EQUAL(addrman.size(), 0U); + BOOST_CHECK_EQUAL(addrman->size(), 0U); // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); - BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), 1U); CService addr1_port = ResolveService("250.1.1.1", 8334); - BOOST_CHECK(addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), 2U); - auto addr_ret2 = addrman.Select().first; + BOOST_CHECK(addrman->Add({CAddress(addr1_port, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), 2U); + auto addr_ret2 = addrman->Select().first; BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333" || addr_ret2.ToString() == "250.1.1.1:8334"); // Test: Add same IP but diff port to tried table; this converts the entry with // the specified port to tried, but not the other. - addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman.size(), 2U); + addrman->Good(CAddress(addr1_port, NODE_NONE)); + BOOST_CHECK_EQUAL(addrman->size(), 2U); bool newOnly = true; - auto addr_ret3 = addrman.Select(newOnly).first; + auto addr_ret3 = addrman->Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_select) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); - BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), 1U); bool newOnly = true; - auto addr_ret1 = addrman.Select(newOnly).first; + auto addr_ret1 = addrman->Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: move addr to tried, select from new expected nothing returned. - BOOST_CHECK(addrman.Good(CAddress(addr1, NODE_NONE))); - BOOST_CHECK_EQUAL(addrman.size(), 1U); - auto addr_ret2 = addrman.Select(newOnly).first; + BOOST_CHECK(addrman->Good(CAddress(addr1, NODE_NONE))); + BOOST_CHECK_EQUAL(addrman->size(), 1U); + auto addr_ret2 = addrman->Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); - auto addr_ret3 = addrman.Select().first; + auto addr_ret3 = addrman->Select().first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK_EQUAL(addrman->size(), 1U); // Add three addresses to new table. @@ -210,65 +165,97 @@ BOOST_AUTO_TEST_CASE(addrman_select) CService addr3 = ResolveService("250.3.2.2", 9999); CService addr4 = ResolveService("250.3.3.3", 9999); - BOOST_CHECK(addrman.Add({CAddress(addr2, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); - BOOST_CHECK(addrman.Add({CAddress(addr3, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); - BOOST_CHECK(addrman.Add({CAddress(addr4, NODE_NONE)}, ResolveService("250.4.1.1", 8333))); + BOOST_CHECK(addrman->Add({CAddress(addr2, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman->Add({CAddress(addr3, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman->Add({CAddress(addr4, NODE_NONE)}, ResolveService("250.4.1.1", 8333))); // Add three addresses to tried table. CService addr5 = ResolveService("250.4.4.4", 8333); CService addr6 = ResolveService("250.4.5.5", 7777); CService addr7 = ResolveService("250.4.6.6", 8333); - BOOST_CHECK(addrman.Add({CAddress(addr5, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); - BOOST_CHECK(addrman.Good(CAddress(addr5, NODE_NONE))); - BOOST_CHECK(addrman.Add({CAddress(addr6, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); - BOOST_CHECK(addrman.Good(CAddress(addr6, NODE_NONE))); - BOOST_CHECK(addrman.Add({CAddress(addr7, NODE_NONE)}, ResolveService("250.1.1.3", 8333))); - BOOST_CHECK(addrman.Good(CAddress(addr7, NODE_NONE))); + BOOST_CHECK(addrman->Add({CAddress(addr5, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman->Good(CAddress(addr5, NODE_NONE))); + BOOST_CHECK(addrman->Add({CAddress(addr6, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman->Good(CAddress(addr6, NODE_NONE))); + BOOST_CHECK(addrman->Add({CAddress(addr7, NODE_NONE)}, ResolveService("250.1.1.3", 8333))); + BOOST_CHECK(addrman->Good(CAddress(addr7, NODE_NONE))); // Test: 6 addrs + 1 addr from last test = 7. - BOOST_CHECK_EQUAL(addrman.size(), 7U); + BOOST_CHECK_EQUAL(addrman->size(), 7U); // Test: Select pulls from new and tried regardless of port number. std::set<uint16_t> ports; for (int i = 0; i < 20; ++i) { - ports.insert(addrman.Select().first.GetPort()); + ports.insert(addrman->Select().first.GetPort()); } BOOST_CHECK_EQUAL(ports.size(), 3U); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); uint32_t num_addrs{0}; - BOOST_CHECK_EQUAL(addrman.size(), num_addrs); + BOOST_CHECK_EQUAL(addrman->size(), num_addrs); while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); - BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); + BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); // Test: No collision in new table yet. - BOOST_CHECK_EQUAL(addrman.size(), num_addrs); + BOOST_CHECK_EQUAL(addrman->size(), num_addrs); } // Test: new table collision! CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); uint32_t collisions{1}; - BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); + BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), num_addrs - collisions); CService addr2 = ResolveService("250.1.1." + ToString(++num_addrs)); - BOOST_CHECK(addrman.Add({CAddress(addr2, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); + BOOST_CHECK(addrman->Add({CAddress(addr2, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), num_addrs - collisions); +} + +BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) +{ + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + CAddress addr{CAddress(ResolveService("253.3.3.3", 8333), NODE_NONE)}; + int64_t start_time{GetAdjustedTime()}; + addr.nTime = start_time; + + // test that multiplicity stays at 1 if nTime doesn't increase + for (unsigned int i = 1; i < 20; ++i) { + std::string addr_ip{ToString(i % 256) + "." + ToString(i >> 8 % 256) + ".1.1"}; + CNetAddr source{ResolveIP(addr_ip)}; + addrman->Add({addr}, source); + } + AddressPosition addr_pos = addrman->FindAddressEntry(addr).value(); + BOOST_CHECK_EQUAL(addr_pos.multiplicity, 1U); + BOOST_CHECK_EQUAL(addrman->size(), 1U); + + // if nTime increases, an addr can occur in up to 8 buckets + // The acceptance probability decreases exponentially with existing multiplicity - + // choose number of iterations such that it gets to 8 with deterministic addrman. + for (unsigned int i = 1; i < 400; ++i) { + std::string addr_ip{ToString(i % 256) + "." + ToString(i >> 8 % 256) + ".1.1"}; + CNetAddr source{ResolveIP(addr_ip)}; + addr.nTime = start_time + i; + addrman->Add({addr}, source); + } + AddressPosition addr_pos_multi = addrman->FindAddressEntry(addr).value(); + BOOST_CHECK_EQUAL(addr_pos_multi.multiplicity, 8U); + // multiplicity doesn't affect size + BOOST_CHECK_EQUAL(addrman->size(), 1U); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) { - auto addrman = std::make_unique<AddrMan>(std::vector<bool>(), /*deterministic=*/true, /*consistency_check_ratio=*/100); + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -296,87 +283,15 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_CHECK(addrman->Good(CAddress(addr2, NODE_NONE))); } -BOOST_AUTO_TEST_CASE(addrman_find) -{ - AddrManTest addrman; - - BOOST_CHECK_EQUAL(addrman.size(), 0U); - - CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); - CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); - CAddress addr3 = CAddress(ResolveService("251.255.2.1", 8333), NODE_NONE); - - CNetAddr source1 = ResolveIP("250.1.2.1"); - CNetAddr source2 = ResolveIP("250.1.2.2"); - - BOOST_CHECK(addrman.Add({addr1}, source1)); - BOOST_CHECK(addrman.Add({addr2}, source2)); - BOOST_CHECK(addrman.Add({addr3}, source1)); - - // Test: ensure Find returns an IP/port matching what we searched on. - AddrInfo* info1 = addrman.Find(addr1); - BOOST_REQUIRE(info1); - BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); - - // Test; Find discriminates by port number. - AddrInfo* info2 = addrman.Find(addr2); - BOOST_REQUIRE(info2); - BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:9999"); - - // Test: Find returns another IP matching what we searched on. - AddrInfo* info3 = addrman.Find(addr3); - BOOST_REQUIRE(info3); - BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); -} - -BOOST_AUTO_TEST_CASE(addrman_create) -{ - AddrManTest addrman; - - BOOST_CHECK_EQUAL(addrman.size(), 0U); - - CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); - CNetAddr source1 = ResolveIP("250.1.2.1"); - - int nId; - AddrInfo* pinfo = addrman.Create(addr1, source1, &nId); - - // Test: The result should be the same as the input addr. - BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); - - AddrInfo* info2 = addrman.Find(addr1); - BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); -} - - -BOOST_AUTO_TEST_CASE(addrman_delete) -{ - AddrManTest addrman; - - BOOST_CHECK_EQUAL(addrman.size(), 0U); - - CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); - CNetAddr source1 = ResolveIP("250.1.2.1"); - - int nId; - addrman.Create(addr1, source1, &nId); - - // Test: Delete should actually delete the addr. - BOOST_CHECK_EQUAL(addrman.size(), 1U); - addrman.Delete(nId); - BOOST_CHECK_EQUAL(addrman.size(), 0U); - AddrInfo* info2 = addrman.Find(addr1); - BOOST_CHECK(info2 == nullptr); -} BOOST_AUTO_TEST_CASE(addrman_getaddr) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); // Test: Sanity check, GetAddr should never return anything if addrman // is empty. - BOOST_CHECK_EQUAL(addrman.size(), 0U); - std::vector<CAddress> vAddr1 = addrman.GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt); + BOOST_CHECK_EQUAL(addrman->size(), 0U); + std::vector<CAddress> vAddr1 = addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt); BOOST_CHECK_EQUAL(vAddr1.size(), 0U); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); @@ -393,18 +308,18 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CNetAddr source2 = ResolveIP("250.2.3.3"); // Test: Ensure GetAddr works with new addresses. - BOOST_CHECK(addrman.Add({addr1, addr3, addr5}, source1)); - BOOST_CHECK(addrman.Add({addr2, addr4}, source2)); + BOOST_CHECK(addrman->Add({addr1, addr3, addr5}, source1)); + BOOST_CHECK(addrman->Add({addr2, addr4}, source2)); - BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 5U); + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 5U); // Net processing asks for 23% of addresses. 23% of 5 is 1 rounded down. - BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt).size(), 1U); + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt).size(), 1U); // Test: Ensure GetAddr works with new and tried addresses. - addrman.Good(CAddress(addr1, NODE_NONE)); - addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 5U); - BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt).size(), 1U); + BOOST_CHECK(addrman->Good(CAddress(addr1, NODE_NONE))); + BOOST_CHECK(addrman->Good(CAddress(addr2, NODE_NONE))); + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 5U); + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt).size(), 1U); // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { @@ -415,24 +330,22 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); - addrman.Add({addr}, ResolveIP(strAddr)); + addrman->Add({addr}, ResolveIP(strAddr)); if (i % 8 == 0) - addrman.Good(addr); + addrman->Good(addr); } - std::vector<CAddress> vAddr = addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); + std::vector<CAddress> vAddr = addrman->GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); - size_t percent23 = (addrman.size() * 23) / 100; + size_t percent23 = (addrman->size() * 23) / 100; BOOST_CHECK_EQUAL(vAddr.size(), percent23); BOOST_CHECK_EQUAL(vAddr.size(), 461U); // (Addrman.size() < number of addresses added) due to address collisions. - BOOST_CHECK_EQUAL(addrman.size(), 2006U); + BOOST_CHECK_EQUAL(addrman->size(), 2006U); } BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { - AddrManTest addrman; - CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -486,8 +399,6 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { - AddrManTest addrman; - CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -564,8 +475,6 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) // 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { - AddrManTest addrman; - CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -619,8 +528,6 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) { - AddrManTest addrman; - CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -700,72 +607,71 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - auto addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); - auto addrman_asmap1_dup = std::make_unique<AddrManTest>(asmap1); - auto addrman_noasmap = std::make_unique<AddrManTest>(); + const auto ratio = GetCheckRatio(m_node); + auto addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); + auto addrman_asmap1_dup = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); + auto addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CNetAddr default_source; - addrman_asmap1->Add({addr}, default_source); stream << *addrman_asmap1; // serizalizing/deserializing addrman with the same asmap stream >> *addrman_asmap1_dup; - std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1->GetBucketAndEntry(addr); - std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup->GetBucketAndEntry(addr); - BOOST_CHECK(bucketAndEntry_asmap1.second != -1); - BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + AddressPosition addr_pos1 = addrman_asmap1->FindAddressEntry(addr).value(); + AddressPosition addr_pos2 = addrman_asmap1_dup->FindAddressEntry(addr).value(); + BOOST_CHECK(addr_pos1.multiplicity != 0); + BOOST_CHECK(addr_pos2.multiplicity != 0); - BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); - BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + BOOST_CHECK(addr_pos1 == addr_pos2); // deserializing asmaped peers.dat to non-asmaped addrman stream << *addrman_asmap1; stream >> *addrman_noasmap; - std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap->GetBucketAndEntry(addr); - BOOST_CHECK(bucketAndEntry_noasmap.second != -1); - BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); - BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + AddressPosition addr_pos3 = addrman_noasmap->FindAddressEntry(addr).value(); + BOOST_CHECK(addr_pos3.multiplicity != 0); + BOOST_CHECK(addr_pos1.bucket != addr_pos3.bucket); + BOOST_CHECK(addr_pos1.position != addr_pos3.position); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); - addrman_noasmap = std::make_unique<AddrManTest>(); + addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); + addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); addrman_noasmap->Add({addr}, default_source); stream << *addrman_noasmap; stream >> *addrman_asmap1; - std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1->GetBucketAndEntry(addr); - BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); - BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); - BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); - BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + AddressPosition addr_pos4 = addrman_asmap1->FindAddressEntry(addr).value(); + BOOST_CHECK(addr_pos4.multiplicity != 0); + BOOST_CHECK(addr_pos4.bucket != addr_pos3.bucket); + BOOST_CHECK(addr_pos4 == addr_pos2); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); - addrman_noasmap = std::make_unique<AddrManTest>(); + addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); + addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); addrman_noasmap->Add({addr, addr2}, default_source); - std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap->GetBucketAndEntry(addr1); - std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap->GetBucketAndEntry(addr2); - BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); - BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + AddressPosition addr_pos5 = addrman_noasmap->FindAddressEntry(addr1).value(); + AddressPosition addr_pos6 = addrman_noasmap->FindAddressEntry(addr2).value(); + BOOST_CHECK(addr_pos5.bucket != addr_pos6.bucket); stream << *addrman_noasmap; stream >> *addrman_asmap1; - std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1->GetBucketAndEntry(addr1); - std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1->GetBucketAndEntry(addr2); - BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); - BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); + AddressPosition addr_pos7 = addrman_asmap1->FindAddressEntry(addr1).value(); + AddressPosition addr_pos8 = addrman_asmap1->FindAddressEntry(addr2).value(); + BOOST_CHECK(addr_pos7.bucket == addr_pos8.bucket); + BOOST_CHECK(addr_pos7.position != addr_pos8.position); } BOOST_AUTO_TEST_CASE(remove_invalid) { // Confirm that invalid addresses are ignored in unserialization. - auto addrman = std::make_unique<AddrManTest>(); + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; @@ -797,29 +703,29 @@ BOOST_AUTO_TEST_CASE(remove_invalid) BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); - addrman = std::make_unique<AddrManTest>(); + addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); stream >> *addrman; BOOST_CHECK_EQUAL(addrman->size(), 2); } BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK(addrman->size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); // Add twenty two addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 23; i++) { CService addr = ResolveService("250.1.1." + ToString(i)); - BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); + BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); // No collisions in tried. - BOOST_CHECK(addrman.Good(addr)); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + BOOST_CHECK(addrman->Good(addr)); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); } // Ensure Good handles duplicates well. @@ -828,114 +734,125 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) CService addr = ResolveService("250.1.1." + ToString(i)); // Unable to add duplicate address to tried table. - BOOST_CHECK(!addrman.Good(addr)); + BOOST_CHECK(!addrman->Good(addr)); // Verify duplicate address not marked as a collision. - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); } } BOOST_AUTO_TEST_CASE(addrman_noevict) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 36; i++) { CService addr = ResolveService("250.1.1." + ToString(i)); - BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); + BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); // No collision yet. - BOOST_CHECK(addrman.Good(addr)); + BOOST_CHECK(addrman->Good(addr)); } // Collision in tried table between 36 and 19. CService addr36 = ResolveService("250.1.1.36"); - BOOST_CHECK(addrman.Add({CAddress(addr36, NODE_NONE)}, source)); - BOOST_CHECK(!addrman.Good(addr36)); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.19:0"); + BOOST_CHECK(addrman->Add({CAddress(addr36, NODE_NONE)}, source)); + BOOST_CHECK(!addrman->Good(addr36)); + BOOST_CHECK_EQUAL(addrman->SelectTriedCollision().first.ToString(), "250.1.1.19:0"); // 36 should be discarded and 19 not evicted. // This means we keep 19 in the tried table and // 36 stays in the new table. - addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + addrman->ResolveCollisions(); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); // Lets create two collisions. for (unsigned int i = 37; i < 59; i++) { CService addr = ResolveService("250.1.1." + ToString(i)); - BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); - BOOST_CHECK(addrman.Good(addr)); + BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); + BOOST_CHECK(addrman->Good(addr)); } // Cause a collision in the tried table. CService addr59 = ResolveService("250.1.1.59"); - BOOST_CHECK(addrman.Add({CAddress(addr59, NODE_NONE)}, source)); - BOOST_CHECK(!addrman.Good(addr59)); + BOOST_CHECK(addrman->Add({CAddress(addr59, NODE_NONE)}, source)); + BOOST_CHECK(!addrman->Good(addr59)); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.10:0"); + BOOST_CHECK_EQUAL(addrman->SelectTriedCollision().first.ToString(), "250.1.1.10:0"); // Cause a second collision in the new table. - BOOST_CHECK(!addrman.Add({CAddress(addr36, NODE_NONE)}, source)); + BOOST_CHECK(!addrman->Add({CAddress(addr36, NODE_NONE)}, source)); // 36 still cannot be moved from new to tried due to colliding with 19 - BOOST_CHECK(!addrman.Good(addr36)); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() != "[::]:0"); + BOOST_CHECK(!addrman->Good(addr36)); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() != "[::]:0"); // Resolve all collisions. - addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + addrman->ResolveCollisions(); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); } BOOST_AUTO_TEST_CASE(addrman_evictionworks) { - AddrManTest addrman; + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK(addrman->size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); // Add 35 addresses CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 36; i++) { CService addr = ResolveService("250.1.1." + ToString(i)); - BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); + BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); // No collision yet. - BOOST_CHECK(addrman.Good(addr)); + BOOST_CHECK(addrman->Good(addr)); } // Collision between 36 and 19. CService addr = ResolveService("250.1.1.36"); - BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); - BOOST_CHECK(!addrman.Good(addr)); + BOOST_CHECK(addrman->Add({CAddress(addr, NODE_NONE)}, source)); + BOOST_CHECK(!addrman->Good(addr)); - auto info = addrman.SelectTriedCollision().first; + auto info = addrman->SelectTriedCollision().first; BOOST_CHECK_EQUAL(info.ToString(), "250.1.1.19:0"); // Ensure test of address fails, so that it is evicted. - addrman.SimConnFail(info); + // Update entry in tried by setting last good connection in the deep past. + BOOST_CHECK(!addrman->Good(info, /*nTime=*/1)); + addrman->Attempt(info, /*fCountFailure=*/false, /*nTime=*/GetAdjustedTime() - 61); // Should swap 36 for 19. - addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + addrman->ResolveCollisions(); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); + AddressPosition addr_pos{addrman->FindAddressEntry(CAddress(addr, NODE_NONE)).value()}; + BOOST_CHECK(addr_pos.tried); // If 36 was swapped for 19, then adding 36 to tried should fail because we // are attempting to add a duplicate. // We check this by verifying Good() returns false and also verifying that // we have no collisions. - BOOST_CHECK(!addrman.Good(addr)); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + BOOST_CHECK(!addrman->Good(addr)); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); // 19 should fail as a collision (not a duplicate) if we now attempt to move // it to the tried table. CService addr19 = ResolveService("250.1.1.19"); - BOOST_CHECK(!addrman.Good(addr19)); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.36:0"); - - addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); + BOOST_CHECK(!addrman->Good(addr19)); + BOOST_CHECK_EQUAL(addrman->SelectTriedCollision().first.ToString(), "250.1.1.36:0"); + + // Eviction is also successful if too much time has passed since last try + SetMockTime(GetTime() + 4 * 60 *60); + addrman->ResolveCollisions(); + BOOST_CHECK(addrman->SelectTriedCollision().first.ToString() == "[::]:0"); + //Now 19 is in tried again, and 36 back to new + AddressPosition addr_pos19{addrman->FindAddressEntry(CAddress(addr19, NODE_NONE)).value()}; + BOOST_CHECK(addr_pos19.tried); + AddressPosition addr_pos36{addrman->FindAddressEntry(CAddress(addr, NODE_NONE)).value()}; + BOOST_CHECK(!addr_pos36.tried); } static CDataStream AddrmanToStream(const AddrMan& addrman) @@ -948,8 +865,7 @@ static CDataStream AddrmanToStream(const AddrMan& addrman) BOOST_AUTO_TEST_CASE(load_addrman) { - AddrMan addrman{/*asmap=*/ std::vector<bool>(), /*deterministic=*/ true, - /*consistency_check_ratio=*/ 100}; + AddrMan addrman{EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)}; CService addr1, addr2, addr3; BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); @@ -968,7 +884,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that the de-serialization does not throw an exception. CDataStream ssPeers1 = AddrmanToStream(addrman); bool exceptionThrown = false; - AddrMan addrman1(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); + AddrMan addrman1{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman1.size() == 0); try { @@ -985,7 +901,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that ReadFromStream creates an addrman with the correct number of addrs. CDataStream ssPeers2 = AddrmanToStream(addrman); - AddrMan addrman2(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); + AddrMan addrman2{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman2.size() == 0); ReadFromStream(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); @@ -1023,7 +939,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that the de-serialization of corrupted peers.dat throws an exception. CDataStream ssPeers1 = MakeCorruptPeersDat(); bool exceptionThrown = false; - AddrMan addrman1(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); + AddrMan addrman1{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; @@ -1039,10 +955,40 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that ReadFromStream fails if peers.dat is corrupt CDataStream ssPeers2 = MakeCorruptPeersDat(); - AddrMan addrman2(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); + AddrMan addrman2{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman2.size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } +BOOST_AUTO_TEST_CASE(addrman_update_address) +{ + // Tests updating nTime via Connected() and nServices via SetServices() + auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + CNetAddr source{ResolveIP("252.2.2.2")}; + CAddress addr{CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE)}; + + int64_t start_time{GetAdjustedTime() - 10000}; + addr.nTime = start_time; + BOOST_CHECK(addrman->Add({addr}, source)); + BOOST_CHECK_EQUAL(addrman->size(), 1U); + + // Updating an addrman entry with a different port doesn't change it + CAddress addr_diff_port{CAddress(ResolveService("250.1.1.1", 8334), NODE_NONE)}; + addr_diff_port.nTime = start_time; + addrman->Connected(addr_diff_port); + addrman->SetServices(addr_diff_port, NODE_NETWORK_LIMITED); + std::vector<CAddress> vAddr1{addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt)}; + BOOST_CHECK_EQUAL(vAddr1.size(), 1U); + BOOST_CHECK_EQUAL(vAddr1.at(0).nTime, start_time); + BOOST_CHECK_EQUAL(vAddr1.at(0).nServices, NODE_NONE); + + // Updating an addrman entry with the correct port is successful + addrman->Connected(addr); + addrman->SetServices(addr, NODE_NETWORK_LIMITED); + std::vector<CAddress> vAddr2 = addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt); + BOOST_CHECK_EQUAL(vAddr2.size(), 1U); + BOOST_CHECK(vAddr2.at(0).nTime >= start_time + 10000); + BOOST_CHECK_EQUAL(vAddr2.at(0).nServices, NODE_NETWORK_LIMITED); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index a7494be882..a923d38467 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -129,11 +129,11 @@ static void shiftArrayRight(unsigned char* to, const unsigned char* from, unsign { unsigned int F = (T+bitsToShift/8); if (F < arrayLength) - to[T] = from[F] >> (bitsToShift%8); + to[T] = uint8_t(from[F] >> (bitsToShift % 8)); else to[T] = 0; if (F + 1 < arrayLength) - to[T] |= from[(F+1)] << (8-bitsToShift%8); + to[T] |= uint8_t(from[(F + 1)] << (8 - bitsToShift % 8)); } } @@ -144,9 +144,9 @@ static void shiftArrayLeft(unsigned char* to, const unsigned char* from, unsigne if (T >= bitsToShift/8) { unsigned int F = T-bitsToShift/8; - to[T] = from[F] << (bitsToShift%8); + to[T] = uint8_t(from[F] << (bitsToShift % 8)); if (T >= bitsToShift/8+1) - to[T] |= from[F-1] >> (8-bitsToShift%8); + to[T] |= uint8_t(from[F - 1] >> (8 - bitsToShift % 8)); } else { to[T] = 0; @@ -202,7 +202,7 @@ BOOST_AUTO_TEST_CASE( unaryOperators ) // ! ~ - BOOST_CHECK(~ZeroL == MaxL); unsigned char TmpArray[32]; - for (unsigned int i = 0; i < 32; ++i) { TmpArray[i] = ~R1Array[i]; } + for (unsigned int i = 0; i < 32; ++i) { TmpArray[i] = uint8_t(~R1Array[i]); } BOOST_CHECK(arith_uint256V(std::vector<unsigned char>(TmpArray,TmpArray+32)) == (~R1L)); BOOST_CHECK(-ZeroL == ZeroL); @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE( unaryOperators ) // ! ~ - // Check if doing _A_ _OP_ _B_ results in the same as applying _OP_ onto each // element of Aarray and Barray, and then converting the result into an arith_uint256. #define CHECKBITWISEOPERATOR(_A_,_B_,_OP_) \ - for (unsigned int i = 0; i < 32; ++i) { TmpArray[i] = _A_##Array[i] _OP_ _B_##Array[i]; } \ + for (unsigned int i = 0; i < 32; ++i) { TmpArray[i] = uint8_t(_A_##Array[i] _OP_ _B_##Array[i]); } \ BOOST_CHECK(arith_uint256V(std::vector<unsigned char>(TmpArray,TmpArray+32)) == (_A_##L _OP_ _B_##L)); #define CHECKASSIGNMENTOPERATOR(_A_,_B_,_OP_) \ diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index a9b08ddc9c..7c502349b3 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -16,6 +16,10 @@ #include <boost/test/unit_test.hpp> +using node::BlockAssembler; +using node::CBlockTemplate; +using node::IncrementExtraNonce; + BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) struct BuildChainTestingSetup : public TestChain100Setup { diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 3921a9d2d1..922fd8e513 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -393,11 +393,11 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // Update the expected result to know about the new output coins assert(tx.vout.size() == 1); const COutPoint outpoint(tx.GetHash(), 0); - result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); + result[outpoint] = Coin{tx.vout[0], int(height), CTransaction(tx).IsCoinBase()}; // Call UpdateCoins on the top cache CTxUndo undo; - UpdateCoins(CTransaction(tx), *(stack.back()), undo, height); + UpdateCoins(CTransaction(tx), *(stack.back()), undo, int(height)); // Update the utxo set for future spends utxoset.insert(outpoint); diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index c2911e5420..92de4ec7ba 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -11,6 +11,8 @@ #include <chrono> +using node::CCoinsStats; +using node::CoinStatsHashType; BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 9b58711eb8..5a3e382c3f 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -42,6 +42,7 @@ constexpr int HARDENED = 2; // Derivation needs access to private keys constexpr int UNSOLVABLE = 4; // This descriptor is not expected to be solvable constexpr int SIGNABLE = 8; // We can sign with this descriptor (this is not true when actual BIP32 derivation is used, as that's not integrated in our signing code) constexpr int DERIVE_HARDENED = 16; // The final derivation is hardened, i.e. ends with *' or *h +constexpr int MIXED_PUBKEYS = 32; /** Compare two descriptors. If only one of them has a checksum, the checksum is ignored. */ bool EqualDescriptor(std::string a, std::string b) @@ -73,6 +74,18 @@ std::string UseHInsteadOfApostrophe(const std::string& desc) return ret; } +// Count the number of times the string "xpub" appears in a descriptor string +static size_t CountXpubs(const std::string& desc) +{ + size_t count = 0; + size_t p = desc.find("xpub", 0); + while (p != std::string::npos) { + count++; + p = desc.find("xpub", p + 1); + } + return count; +} + const std::set<std::vector<uint32_t>> ONLY_EMPTY{{}}; void DoCheck(const std::string& prv, const std::string& pub, const std::string& norm_prv, const std::string& norm_pub, int flags, const std::vector<std::vector<std::string>>& scripts, const std::optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, @@ -171,7 +184,8 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& // Check whether keys are in the cache const auto& der_xpub_cache = desc_cache.GetCachedDerivedExtPubKeys(); const auto& parent_xpub_cache = desc_cache.GetCachedParentExtPubKeys(); - if ((flags & RANGE) && !(flags & DERIVE_HARDENED)) { + const size_t num_xpubs = CountXpubs(pub1); + if ((flags & RANGE) && !(flags & (DERIVE_HARDENED))) { // For ranged, unhardened derivation, None of the keys in origins should appear in the cache but the cache should have parent keys // But we can derive one level from each of those parent keys and find them all BOOST_CHECK(der_xpub_cache.empty()); @@ -183,13 +197,22 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& xpub.Derive(der, i); pubkeys.insert(der.pubkey); } + int count_pks = 0; for (const auto& origin_pair : script_provider_cached.origins) { const CPubKey& pk = origin_pair.second.first; - BOOST_CHECK(pubkeys.count(pk) > 0); + count_pks += pubkeys.count(pk); } - } else if (pub1.find("xpub") != std::string::npos) { + if (flags & MIXED_PUBKEYS) { + BOOST_CHECK_EQUAL(num_xpubs, count_pks); + } else { + BOOST_CHECK_EQUAL(script_provider_cached.origins.size(), count_pks); + } + } else if (num_xpubs > 0) { // For ranged, hardened derivation, or not ranged, but has an xpub, all of the keys should appear in the cache - BOOST_CHECK(der_xpub_cache.size() + parent_xpub_cache.size() == script_provider_cached.origins.size()); + BOOST_CHECK(der_xpub_cache.size() + parent_xpub_cache.size() == num_xpubs); + if (!(flags & MIXED_PUBKEYS)) { + BOOST_CHECK(num_xpubs == script_provider_cached.origins.size()); + } // Get all of the derived pubkeys std::set<CPubKey> pubkeys; for (const auto& xpub_map_pair : der_xpub_cache) { @@ -206,12 +229,18 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& xpub.Derive(der, i); pubkeys.insert(der.pubkey); } + int count_pks = 0; for (const auto& origin_pair : script_provider_cached.origins) { const CPubKey& pk = origin_pair.second.first; - BOOST_CHECK(pubkeys.count(pk) > 0); + count_pks += pubkeys.count(pk); } - } else { - // No xpub, nothing should be cached + if (flags & MIXED_PUBKEYS) { + BOOST_CHECK_EQUAL(num_xpubs, count_pks); + } else { + BOOST_CHECK_EQUAL(script_provider_cached.origins.size(), count_pks); + } + } else if (!(flags & MIXED_PUBKEYS)) { + // Only const pubkeys, nothing should be cached BOOST_CHECK(der_xpub_cache.empty()); BOOST_CHECK(parent_xpub_cache.empty()); } @@ -333,6 +362,11 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", "wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); Check("combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", "combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", RANGE, {{"2102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e076ac","76a914f90e3178ca25f2c808dc76624032d352fdbdfaf288ac","0014f90e3178ca25f2c808dc76624032d352fdbdfaf2","a91408f3ea8c68d4a7585bf9e8bda226723f70e445f087"},{"21032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ecac","76a914a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b788ac","0014a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b7","a91473e39884cb71ae4e5ac9739e9225026c99763e6687"}}, std::nullopt, {{0}, {1}}); + // Mixed xpubs and const pubkeys + Check("wsh(multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))","wsh(multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))","wsh(multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))","wsh(multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", MIXED_PUBKEYS, {{"0020cb155486048b23a6da976d4c6fe071a2dbc8a7b57aaf225b8955f2e2a27b5f00"}},OutputType::BECH32,{{0},{}}); + // Mixed range xpubs and const pubkeys + Check("multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)","multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)","multi(1,xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)","multi(1,xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", RANGE | MIXED_PUBKEYS, {{"512102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e0762103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"},{"5121032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ec2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"},{"5121035d30b6c66dc1e036c45369da8287518cf7e0d6ed1e2b905171c605708f14ca032103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd52ae"}}, std::nullopt,{{2},{1},{0},{}}); + CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "Key path value 2147483648 is out of range"); // BIP 32 path element overflow CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "Key path value '1aa' is not a valid uint32"); // Path is not valid uint diff --git a/src/test/fuzz/addition_overflow.cpp b/src/test/fuzz/addition_overflow.cpp index c6cfbd8d30..cfad41659e 100644 --- a/src/test/fuzz/addition_overflow.cpp +++ b/src/test/fuzz/addition_overflow.cpp @@ -5,6 +5,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/overflow.h> #include <cstdint> #include <string> diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 9c85c20e2b..3699abb597 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -11,8 +11,10 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/setup_common.h> #include <time.h> #include <util/asmap.h> +#include <util/system.h> #include <cassert> #include <cstdint> @@ -20,16 +22,26 @@ #include <string> #include <vector> +namespace { +const BasicTestingSetup* g_setup; + +int32_t GetCheckRatio() +{ + return std::clamp<int32_t>(g_setup->m_node.args->GetIntArg("-checkaddrman", 0), 0, 1000000); +} +} // namespace + void initialize_addrman() { - SelectParams(CBaseChainParams::REGTEST); + static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::REGTEST); + g_setup = testing_setup.get(); } FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - AddrMan addr_man(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); + AddrMan addr_man{/*asmap=*/std::vector<bool>(), /*deterministic=*/false, GetCheckRatio()}; try { ReadFromStream(addr_man, data_stream); } catch (const std::exception&) { @@ -113,7 +125,7 @@ class AddrManDeterministic : public AddrMan { public: explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) - : AddrMan(std::move(asmap), /*deterministic=*/true, /*consistency_check_ratio=*/0) + : AddrMan{std::move(asmap), /*deterministic=*/true, GetCheckRatio()} { WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); } diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 2f33598348..994b4b9e49 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -26,6 +26,10 @@ #include <string> #include <vector> +using node::CCoinsStats; +using node::CoinStatsHashType; +using node::GetUTXOStats; + namespace { const TestingSetup* g_setup; const Coin EMPTY_COIN{}; @@ -269,7 +273,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_setup->m_node.chainman->m_blockman)), stats); + (void)GetUTXOStats(&coins_view_cache, g_setup->m_node.chainman->m_blockman, stats); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index f87b6f1503..a14d28f4ef 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -12,21 +12,29 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> +#include <util/system.h> #include <util/translation.h> #include <cstdint> #include <vector> +namespace { +const BasicTestingSetup* g_setup; +} // namespace + void initialize_connman() { static const auto testing_setup = MakeNoLogFileContext<>(); + g_setup = testing_setup.get(); } FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - AddrMan addrman(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); + AddrMan addrman(/*asmap=*/std::vector<bool>(), + /*deterministic=*/false, + g_setup->m_node.args->GetIntArg("-checkaddrman", 0)); CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); @@ -90,12 +98,6 @@ FUZZ_TARGET_INIT(connman, initialize_connman) (void)connman.OutboundTargetReached(fuzzed_data_provider.ConsumeBool()); }, [&] { - // Limit now to int32_t to avoid signed integer overflow - (void)connman.PoissonNextSendInbound( - std::chrono::microseconds{fuzzed_data_provider.ConsumeIntegral<int32_t>()}, - std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<int>()}); - }, - [&] { CSerializedNetMsg serialized_net_msg; serialized_net_msg.m_type = fuzzed_data_provider.ConsumeRandomLengthString(CMessageHeader::COMMAND_SIZE); serialized_net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index 5e60b0f25b..596614a71b 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/overflow.h> #include <cassert> #include <cstdint> diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 48574d71cc..ed6f172a2a 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -22,7 +22,9 @@ #include <pubkey.h> #include <script/keyorigin.h> #include <streams.h> +#include <test/util/setup_common.h> #include <undo.h> +#include <util/system.h> #include <version.h> #include <exception> @@ -33,8 +35,17 @@ #include <test/fuzz/fuzz.h> +using node::SnapshotMetadata; + +namespace { +const BasicTestingSetup* g_setup; +} // namespace + void initialize_deserialize() { + static const auto testing_setup = MakeNoLogFileContext<>(); + g_setup = testing_setup.get(); + // Fuzzers using pubkey must hold an ECCVerifyHandle. static const ECCVerifyHandle verify_handle; } @@ -189,7 +200,9 @@ FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - AddrMan am(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); + AddrMan am(/*asmap=*/std::vector<bool>(), + /*deterministic=*/false, + g_setup->m_node.args->GetIntArg("-checkaddrman", 0)); DeserializeFromFuzzingInput(buffer, am); }) FUZZ_TARGET_DESERIALIZE(blockheader_deserialize, { diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a33297e0ed..e9debd8c45 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -12,6 +12,7 @@ #include <cstdint> #include <exception> +#include <functional> #include <memory> #include <string> #include <unistd.h> @@ -19,6 +20,29 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; +/** + * A copy of the command line arguments that start with `--`. + * First `LLVMFuzzerInitialize()` is called, which saves the arguments to `g_args`. + * Later, depending on the fuzz test, `G_TEST_COMMAND_LINE_ARGUMENTS()` may be + * called by `BasicTestingSetup` constructor to fetch those arguments and store + * them in `BasicTestingSetup::m_node::args`. + */ +static std::vector<const char*> g_args; + +static void SetArgs(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + // Only take into account arguments that start with `--`. The others are for the fuzz engine: + // `fuzz -runs=1 fuzz_seed_corpus/address_deserialize_v2 --checkaddrman=5` + if (strlen(argv[i]) > 2 && argv[i][0] == '-' && argv[i][1] == '-') { + g_args.push_back(argv[i]); + } + } +} + +const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = []() { + return g_args; +}; + std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>>& FuzzTargets() { static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>> g_fuzz_targets; @@ -95,6 +119,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) // This function is used by libFuzzer extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { + SetArgs(*argc, *argv); initialize(); return 0; } diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index d6879067b8..b4bb4c6dc6 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -19,27 +19,13 @@ #include <vector> namespace { -uint64_t MapIntoRange(const uint64_t x, const uint64_t n) -{ - const uint64_t x_hi = x >> 32; - const uint64_t x_lo = x & 0xFFFFFFFF; - const uint64_t n_hi = n >> 32; - const uint64_t n_lo = n & 0xFFFFFFFF; - const uint64_t ac = x_hi * n_hi; - const uint64_t ad = x_hi * n_lo; - const uint64_t bc = x_lo * n_hi; - const uint64_t bd = x_lo * n_lo; - const uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); - const uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); - return upper64; -} uint64_t HashToRange(const std::vector<uint8_t>& element, const uint64_t f) { const uint64_t hash = CSipHasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL) .Write(element.data(), element.size()) .Finalize(); - return MapIntoRange(hash, f); + return FastRange64(hash, f); } std::vector<uint64_t> BuildHashedSet(const std::unordered_set<std::vector<uint8_t>, ByteVectorHash>& elements, const uint64_t f) diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 33b0d7325b..3087f11771 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -26,6 +26,7 @@ #include <univalue.h> #include <util/check.h> #include <util/moneystr.h> +#include <util/overflow.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> diff --git a/src/test/fuzz/minisketch.cpp b/src/test/fuzz/minisketch.cpp index 93954bd3cf..a17be73f6c 100644 --- a/src/test/fuzz/minisketch.cpp +++ b/src/test/fuzz/minisketch.cpp @@ -12,6 +12,8 @@ #include <map> #include <numeric> +using node::MakeMinisketch32; + FUZZ_TARGET(minisketch) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index a7b2b8bfc1..88c22ca305 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -70,13 +70,13 @@ FUZZ_TARGET_INIT(p2p_transport_serialization, initialize_p2p_transport_serializa const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; bool reject_message{false}; CNetMessage msg = deserializer.GetMessage(m_time, reject_message); - assert(msg.m_command.size() <= CMessageHeader::COMMAND_SIZE); + assert(msg.m_type.size() <= CMessageHeader::COMMAND_SIZE); assert(msg.m_raw_message_size <= mutable_msg_bytes.size()); assert(msg.m_raw_message_size == CMessageHeader::HEADER_SIZE + msg.m_message_size); assert(msg.m_time == m_time); std::vector<unsigned char> header; - auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_command, MakeUCharSpan(msg.m_recv)); + auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_type, MakeUCharSpan(msg.m_recv)); serializer.prepareForTransport(msg2, header); } } diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 1123c8c170..0004d82d66 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -9,6 +9,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/overflow.h> #include <cstdint> #include <optional> diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index f221e2ba67..669688a80d 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -18,6 +18,10 @@ #include <string> #include <vector> +using node::AnalyzePSBT; +using node::PSBTAnalysis; +using node::PSBTInputAnalysis; + void initialize_psbt() { static const ECCVerifyHandle verify_handle; diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index deffe26b17..f6c591aca4 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -34,7 +34,7 @@ public: return m_fuzzed_data_provider.ConsumeBool(); } - bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, const ScriptExecutionData& execdata, ScriptError* serror = nullptr) const override + bool CheckSchnorrSignature(Span<const unsigned char> sig, Span<const unsigned char> pubkey, SigVersion sigversion, ScriptExecutionData& execdata, ScriptError* serror = nullptr) const override { return m_fuzzed_data_provider.ConsumeBool(); } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index ab646c68fc..8f071b71fe 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -276,20 +276,14 @@ FUZZ_TARGET(string) } { - const int atoi_result = atoi(random_string_1.c_str()); const int locale_independent_atoi_result = LocaleIndependentAtoi<int>(random_string_1); const int64_t atoi64_result = atoi64_legacy(random_string_1); - const bool out_of_range = atoi64_result < std::numeric_limits<int>::min() || atoi64_result > std::numeric_limits<int>::max(); - if (out_of_range) { - assert(locale_independent_atoi_result == 0); - } else { - assert(atoi_result == locale_independent_atoi_result); - } + assert(locale_independent_atoi_result == std::clamp<int64_t>(atoi64_result, std::numeric_limits<int>::min(), std::numeric_limits<int>::max())); } { const int64_t atoi64_result = atoi64_legacy(random_string_1); const int64_t locale_independent_atoi_result = LocaleIndependentAtoi<int64_t>(random_string_1); - assert(atoi64_result == locale_independent_atoi_result || locale_independent_atoi_result == 0); + assert(atoi64_result == locale_independent_atoi_result); } } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index fe1b9c7c0c..df5b271d06 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -14,6 +14,8 @@ #include <validation.h> #include <validationinterface.h> +using node::BlockAssembler; + namespace { const TestingSetup* g_setup; diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 843b29b911..47c2be3faa 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -8,10 +8,13 @@ #include <pubkey.h> #include <test/fuzz/util.h> #include <test/util/script.h> +#include <util/overflow.h> #include <util/rbf.h> #include <util/time.h> #include <version.h> +#include <memory> + FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider{fuzzed_data_provider} { @@ -157,6 +160,20 @@ int FuzzedSock::Connect(const sockaddr*, socklen_t) const return 0; } +std::unique_ptr<Sock> FuzzedSock::Accept(sockaddr* addr, socklen_t* addr_len) const +{ + constexpr std::array accept_errnos{ + ECONNABORTED, + EINTR, + ENOMEM, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, accept_errnos); + return std::unique_ptr<FuzzedSock>(); + } + return std::make_unique<FuzzedSock>(m_fuzzed_data_provider); +} + int FuzzedSock::GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const { constexpr std::array getsockopt_errnos{ @@ -264,8 +281,8 @@ CAmount ConsumeMoney(FuzzedDataProvider& fuzzed_data_provider, const std::option int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min, const std::optional<int64_t>& max) noexcept { // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) disables mocktime. - static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:01Z"); - static const int64_t time_max = ParseISO8601DateTime("9999-12-31T23:59:59Z"); + static const int64_t time_min{ParseISO8601DateTime("2000-01-01T00:00:01Z")}; + static const int64_t time_max{ParseISO8601DateTime("2100-12-31T23:59:59Z")}; return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(min.value_or(time_min), max.value_or(time_max)); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 7937315822..fd7f40c01d 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -193,17 +193,6 @@ template <typename T> } } -template <class T> -[[nodiscard]] bool AdditionOverflow(const T i, const T j) noexcept -{ - static_assert(std::is_integral<T>::value, "Integral required."); - if (std::numeric_limits<T>::is_signed) { - return (i > 0 && j > std::numeric_limits<T>::max() - i) || - (i < 0 && j < std::numeric_limits<T>::min() - i); - } - return std::numeric_limits<T>::max() - i < j; -} - [[nodiscard]] bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept; /** @@ -412,6 +401,8 @@ public: int Connect(const sockaddr*, socklen_t) const override; + std::unique_ptr<Sock> Accept(sockaddr* addr, socklen_t* addr_len) const override; + int GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const override; bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 1b9f0c8a02..19ca1f0c99 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -13,6 +13,8 @@ #include <validation.h> #include <validationinterface.h> +using node::SnapshotMetadata; + namespace { const std::vector<std::shared_ptr<CBlock>>* g_chain; diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 88ce9648bc..d5142c8d74 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -6,6 +6,7 @@ #include <util/strencodings.h> #include <util/system.h> +#include <limits> #include <string> #include <utility> #include <vector> @@ -144,6 +145,11 @@ BOOST_AUTO_TEST_CASE(intarg) BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 11), 0); BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 0); + // Check under-/overflow behavior. + ResetArgs("-foo=-9223372036854775809 -bar=9223372036854775808"); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 0), std::numeric_limits<int64_t>::min()); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 0), std::numeric_limits<int64_t>::max()); + ResetArgs("-foo=11 -bar=12"); BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 0), 11); BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 12); diff --git a/src/test/main.cpp b/src/test/main.cpp index 5885564074..1ad8fcce3a 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -11,6 +11,7 @@ #include <test/util/setup_common.h> +#include <functional> #include <iostream> /** Redirect debug log to unit_test.log files */ @@ -24,3 +25,17 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN = [](const std::str if (!should_log) return; std::cout << s; }; + +/** + * Retrieve the command line arguments from boost. + * Allows usage like: + * `test_bitcoin --run_test="net_tests/cnode_listen_port" -- -checkaddrman=1 -printtoconsole=1` + * which would return `["-checkaddrman=1", "-printtoconsole=1"]`. + */ +const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = []() { + std::vector<const char*> args; + for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) { + args.push_back(boost::unit_test::framework::master_test_suite().argv[i]); + } + return args; +}; diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index bdc6ff6130..e07eb95856 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -24,6 +24,9 @@ #include <boost/test/unit_test.hpp> +using node::BlockAssembler; +using node::CBlockTemplate; + namespace miner_tests { struct MinerTestingSetup : public TestingSetup { void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); diff --git a/src/test/minisketch_tests.cpp b/src/test/minisketch_tests.cpp index f7dd18923b..9c53ace633 100644 --- a/src/test/minisketch_tests.cpp +++ b/src/test/minisketch_tests.cpp @@ -11,6 +11,8 @@ #include <utility> +using node::MakeMinisketch32; + BOOST_AUTO_TEST_SUITE(minisketch_tests) BOOST_AUTO_TEST_CASE(minisketch_test) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 86786af450..b0befe2f58 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -593,7 +593,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) // that a normal IPv4 address is among the entries, but if this address is // !IsRoutable the undefined behavior is easier to trigger deterministically { - LOCK(cs_mapLocalHost); + LOCK(g_maplocalhost_mutex); in_addr ipv4AddrLocal; ipv4AddrLocal.s_addr = 0x0100007f; CNetAddr addr = CNetAddr(ipv4AddrLocal); diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index d5a4d3fd80..2f43ae52f7 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -73,7 +73,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_overflow_target) { const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); uint256 hash; - unsigned int nBits = ~0x00800000; + unsigned int nBits{~0x00800000U}; hash.SetHex("0x1"); BOOST_CHECK(!CheckProofOfWork(hash, nBits, consensus)); } diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 12c5848eaf..89814748fe 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -220,7 +220,7 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) prevector_tester<8, int> test; for (int i = 0; i < 2048; i++) { if (InsecureRandBits(2) == 0) { - test.insert(InsecureRandRange(test.size() + 1), InsecureRand32()); + test.insert(InsecureRandRange(test.size() + 1), int(InsecureRand32())); } if (test.size() > 0 && InsecureRandBits(2) == 1) { test.erase(InsecureRandRange(test.size())); @@ -230,7 +230,7 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) test.resize(new_size); } if (InsecureRandBits(3) == 3) { - test.insert(InsecureRandRange(test.size() + 1), 1 + InsecureRandBool(), InsecureRand32()); + test.insert(InsecureRandRange(test.size() + 1), 1 + InsecureRandBool(), int(InsecureRand32())); } if (InsecureRandBits(3) == 4) { int del = std::min<int>(test.size(), 1 + (InsecureRandBool())); @@ -238,7 +238,7 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) test.erase(beg, beg + del); } if (InsecureRandBits(4) == 5) { - test.push_back(InsecureRand32()); + test.push_back(int(InsecureRand32())); } if (test.size() > 0 && InsecureRandBits(4) == 6) { test.pop_back(); @@ -247,7 +247,7 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) int values[4]; int num = 1 + (InsecureRandBits(2)); for (int k = 0; k < num; k++) { - values[k] = InsecureRand32(); + values[k] = int(InsecureRand32()); } test.insert_range(InsecureRandRange(test.size() + 1), values, values + num); } @@ -263,13 +263,13 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) test.shrink_to_fit(); } if (test.size() > 0) { - test.update(InsecureRandRange(test.size()), InsecureRand32()); + test.update(InsecureRandRange(test.size()), int(InsecureRand32())); } if (InsecureRandBits(10) == 11) { test.clear(); } if (InsecureRandBits(9) == 12) { - test.assign(InsecureRandBits(5), InsecureRand32()); + test.assign(InsecureRandBits(5), int(InsecureRand32())); } if (InsecureRandBits(3) == 3) { test.swap(); @@ -283,8 +283,8 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) if (InsecureRandBits(5) == 19) { unsigned int num = 1 + (InsecureRandBits(4)); std::vector<int> values(num); - for (auto &v : values) { - v = InsecureRand32(); + for (int& v : values) { + v = int(InsecureRand32()); } test.resize_uninitialized(values); } diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index c16087bc1b..1601b02356 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -91,8 +91,9 @@ void static RandomScript(CScript &script) { script << oplist[InsecureRandRange(std::size(oplist))]; } -void static RandomTransaction(CMutableTransaction &tx, bool fSingle) { - tx.nVersion = InsecureRand32(); +void static RandomTransaction(CMutableTransaction& tx, bool fSingle) +{ + tx.nVersion = int(InsecureRand32()); tx.vin.clear(); tx.vout.clear(); tx.nLockTime = (InsecureRandBool()) ? InsecureRand32() : 0; @@ -126,7 +127,7 @@ BOOST_AUTO_TEST_CASE(sighash_test) int nRandomTests = 50000; #endif for (int i=0; i<nRandomTests; i++) { - int nHashType = InsecureRand32(); + int nHashType{int(InsecureRand32())}; CMutableTransaction txTo; RandomTransaction(txTo, (nHashType & 0x1f) == SIGHASH_SINGLE); CScript scriptCode; diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 7ede79279f..6dadf09176 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -114,8 +114,8 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_test) } else { // randomly choose something in the range [MTP, MTP*2] int64_t medianTimePast = vBlocksMain[i].GetMedianTimePast(); - int r = InsecureRandRange(medianTimePast); - vBlocksMain[i].nTime = r + medianTimePast; + int r{int(InsecureRandRange(medianTimePast))}; + vBlocksMain[i].nTime = uint32_t(r + medianTimePast); vBlocksMain[i].nTimeMax = std::max(vBlocksMain[i].nTime, vBlocksMain[i-1].nTimeMax); } } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 1fe51fadd4..4fb7f9c405 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -220,7 +220,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) fValid = false; break; } - COutPoint outpoint(uint256S(vinput[0].get_str()), vinput[1].get_int()); + COutPoint outpoint{uint256S(vinput[0].get_str()), uint32_t(vinput[1].get_int())}; mapprevOutScriptPubKeys[outpoint] = ParseScript(vinput[2].get_str()); if (vinput.size() >= 4) { @@ -308,7 +308,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) fValid = false; break; } - COutPoint outpoint(uint256S(vinput[0].get_str()), vinput[1].get_int()); + COutPoint outpoint{uint256S(vinput[0].get_str()), uint32_t(vinput[1].get_int())}; mapprevOutScriptPubKeys[outpoint] = ParseScript(vinput[2].get_str()); if (vinput.size() >= 4) { diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 6f78b43826..560efb6b42 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -327,4 +327,236 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } } + +// Tests for packages containing transactions that have same-txid-different-witness equivalents in +// the mempool. +BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) +{ + // Mine blocks to mature coinbases. + mineBlocks(5); + LOCK(cs_main); + + // Transactions with a same-txid-different-witness transaction in the mempool should be ignored, + // and the mempool entry's wtxid returned. + CScript witnessScript = CScript() << OP_DROP << OP_TRUE; + CScript scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[0], /*vout=*/ 0, + /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, + /*output_destination=*/ scriptPubKey, + /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ false); + CTransactionRef ptx_parent = MakeTransactionRef(mtx_parent); + + // Make two children with the same txid but different witnesses. + CScriptWitness witness1; + witness1.stack.push_back(std::vector<unsigned char>(1)); + witness1.stack.push_back(std::vector<unsigned char>(witnessScript.begin(), witnessScript.end())); + + CScriptWitness witness2(witness1); + witness2.stack.push_back(std::vector<unsigned char>(2)); + witness2.stack.push_back(std::vector<unsigned char>(witnessScript.begin(), witnessScript.end())); + + CKey child_key; + child_key.MakeNewKey(true); + CScript child_locking_script = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); + CMutableTransaction mtx_child1; + mtx_child1.nVersion = 1; + mtx_child1.vin.resize(1); + mtx_child1.vin[0].prevout.hash = ptx_parent->GetHash(); + mtx_child1.vin[0].prevout.n = 0; + mtx_child1.vin[0].scriptSig = CScript(); + mtx_child1.vin[0].scriptWitness = witness1; + mtx_child1.vout.resize(1); + mtx_child1.vout[0].nValue = CAmount(48 * COIN); + mtx_child1.vout[0].scriptPubKey = child_locking_script; + + CMutableTransaction mtx_child2{mtx_child1}; + mtx_child2.vin[0].scriptWitness = witness2; + + CTransactionRef ptx_child1 = MakeTransactionRef(mtx_child1); + CTransactionRef ptx_child2 = MakeTransactionRef(mtx_child2); + + // child1 and child2 have the same txid + BOOST_CHECK_EQUAL(ptx_child1->GetHash(), ptx_child2->GetHash()); + // child1 and child2 have different wtxids + BOOST_CHECK(ptx_child1->GetWitnessHash() != ptx_child2->GetWitnessHash()); + + // Try submitting Package1{parent, child1} and Package2{parent, child2} where the children are + // same-txid-different-witness. + { + const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + {ptx_parent, ptx_child1}, /*test_accept=*/ false); + BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), + "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); + auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child1 = submit_witness1.m_tx_results.find(ptx_child1->GetWitnessHash()); + BOOST_CHECK(it_parent1 != submit_witness1.m_tx_results.end()); + BOOST_CHECK_MESSAGE(it_parent1->second.m_state.IsValid(), + "Transaction unexpectedly failed: " << it_parent1->second.m_state.GetRejectReason()); + BOOST_CHECK(it_child1 != submit_witness1.m_tx_results.end()); + BOOST_CHECK_MESSAGE(it_child1->second.m_state.IsValid(), + "Transaction unexpectedly failed: " << it_child1->second.m_state.GetRejectReason()); + + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent->GetHash()))); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); + + const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + {ptx_parent, ptx_child2}, /*test_accept=*/ false); + BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), + "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); + auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); + BOOST_CHECK(it_parent2_deduped != submit_witness2.m_tx_results.end()); + BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child2 != submit_witness2.m_tx_results.end()); + BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); + + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); + BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); + } + + // Try submitting Package1{child2, grandchild} where child2 is same-txid-different-witness as + // the in-mempool transaction, child1. Since child1 exists in the mempool and its outputs are + // available, child2 should be ignored and grandchild should be accepted. + // + // This tests a potential censorship vector in which an attacker broadcasts a competing package + // where a parent's witness is mutated. The honest package should be accepted despite the fact + // that we don't allow witness replacement. + CKey grandchild_key; + grandchild_key.MakeNewKey(true); + CScript grandchild_locking_script = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); + auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/ ptx_child2, /* vout=*/ 0, + /*input_height=*/ 0, /*input_signing_key=*/ child_key, + /*output_destination=*/ grandchild_locking_script, + /*output_amount=*/ CAmount(47 * COIN), /*submit=*/ false); + CTransactionRef ptx_grandchild = MakeTransactionRef(mtx_grandchild); + + // We already submitted child1 above. + { + const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, + {ptx_child2, ptx_grandchild}, /*test_accept=*/ false); + BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), + "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); + auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); + auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); + BOOST_CHECK(it_child2_ignored != submit_spend_ignored.m_tx_results.end()); + BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_grandchild != submit_spend_ignored.m_tx_results.end()); + BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); + BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Wtxid(ptx_grandchild->GetWitnessHash()))); + } + + // A package Package{parent1, parent2, parent3, child} where the parents are a mixture of + // identical-tx-in-mempool, same-txid-different-witness-in-mempool, and new transactions. + Package package_mixed; + + // Give all the parents anyone-can-spend scripts so we don't have to deal with signing the child. + CScript acs_script = CScript() << OP_TRUE; + CScript acs_spk = GetScriptForDestination(WitnessV0ScriptHash(acs_script)); + CScriptWitness acs_witness; + acs_witness.stack.push_back(std::vector<unsigned char>(acs_script.begin(), acs_script.end())); + + // parent1 will already be in the mempool + auto mtx_parent1 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[1], /*vout=*/ 0, + /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, + /*output_destination=*/ acs_spk, + /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ true); + CTransactionRef ptx_parent1 = MakeTransactionRef(mtx_parent1); + package_mixed.push_back(ptx_parent1); + + // parent2 will have a same-txid-different-witness tx already in the mempool + CScript grandparent2_script = CScript() << OP_DROP << OP_TRUE; + CScript grandparent2_spk = GetScriptForDestination(WitnessV0ScriptHash(grandparent2_script)); + CScriptWitness parent2_witness1; + parent2_witness1.stack.push_back(std::vector<unsigned char>(1)); + parent2_witness1.stack.push_back(std::vector<unsigned char>(grandparent2_script.begin(), grandparent2_script.end())); + CScriptWitness parent2_witness2; + parent2_witness2.stack.push_back(std::vector<unsigned char>(2)); + parent2_witness2.stack.push_back(std::vector<unsigned char>(grandparent2_script.begin(), grandparent2_script.end())); + + // Create grandparent2 creating an output with multiple spending paths. Submit to mempool. + auto mtx_grandparent2 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[2], /* vout=*/ 0, + /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, + /*output_destination=*/ grandparent2_spk, + /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ true); + CTransactionRef ptx_grandparent2 = MakeTransactionRef(mtx_grandparent2); + + CMutableTransaction mtx_parent2_v1; + mtx_parent2_v1.nVersion = 1; + mtx_parent2_v1.vin.resize(1); + mtx_parent2_v1.vin[0].prevout.hash = ptx_grandparent2->GetHash(); + mtx_parent2_v1.vin[0].prevout.n = 0; + mtx_parent2_v1.vin[0].scriptSig = CScript(); + mtx_parent2_v1.vin[0].scriptWitness = parent2_witness1; + mtx_parent2_v1.vout.resize(1); + mtx_parent2_v1.vout[0].nValue = CAmount(48 * COIN); + mtx_parent2_v1.vout[0].scriptPubKey = acs_spk; + + CMutableTransaction mtx_parent2_v2{mtx_parent2_v1}; + mtx_parent2_v2.vin[0].scriptWitness = parent2_witness2; + + CTransactionRef ptx_parent2_v1 = MakeTransactionRef(mtx_parent2_v1); + CTransactionRef ptx_parent2_v2 = MakeTransactionRef(mtx_parent2_v2); + // Put parent2_v1 in the package, submit parent2_v2 to the mempool. + const MempoolAcceptResult parent2_v2_result = m_node.chainman->ProcessTransaction(ptx_parent2_v2); + BOOST_CHECK(parent2_v2_result.m_result_type == MempoolAcceptResult::ResultType::VALID); + package_mixed.push_back(ptx_parent2_v1); + + // parent3 will be a new transaction + auto mtx_parent3 = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[3], /*vout=*/ 0, + /*input_height=*/ 0, /*input_signing_key=*/ coinbaseKey, + /*output_destination=*/ acs_spk, + /*output_amount=*/ CAmount(49 * COIN), /*submit=*/ false); + CTransactionRef ptx_parent3 = MakeTransactionRef(mtx_parent3); + package_mixed.push_back(ptx_parent3); + + // child spends parent1, parent2, and parent3 + CKey mixed_grandchild_key; + mixed_grandchild_key.MakeNewKey(true); + CScript mixed_child_spk = GetScriptForDestination(WitnessV0KeyHash(mixed_grandchild_key.GetPubKey())); + + CMutableTransaction mtx_mixed_child; + mtx_mixed_child.vin.push_back(CTxIn(COutPoint(ptx_parent1->GetHash(), 0))); + mtx_mixed_child.vin.push_back(CTxIn(COutPoint(ptx_parent2_v1->GetHash(), 0))); + mtx_mixed_child.vin.push_back(CTxIn(COutPoint(ptx_parent3->GetHash(), 0))); + mtx_mixed_child.vin[0].scriptWitness = acs_witness; + mtx_mixed_child.vin[1].scriptWitness = acs_witness; + mtx_mixed_child.vin[2].scriptWitness = acs_witness; + mtx_mixed_child.vout.push_back(CTxOut(145 * COIN, mixed_child_spk)); + CTransactionRef ptx_mixed_child = MakeTransactionRef(mtx_mixed_child); + package_mixed.push_back(ptx_mixed_child); + + // Submit package: + // parent1 should be ignored + // parent2_v1 should be ignored (and v2 wtxid returned) + // parent3 should be accepted + // child should be accepted + { + const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false); + BOOST_CHECK_MESSAGE(mixed_result.m_state.IsValid(), mixed_result.m_state.GetRejectReason()); + auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); + auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); + auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); + auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); + BOOST_CHECK(it_parent1 != mixed_result.m_tx_results.end()); + BOOST_CHECK(it_parent2 != mixed_result.m_tx_results.end()); + BOOST_CHECK(it_parent3 != mixed_result.m_tx_results.end()); + BOOST_CHECK(it_child != mixed_result.m_tx_results.end()); + + BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); + + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent1->GetHash()))); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent2_v1->GetHash()))); + BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_parent2_v1->GetWitnessHash()))); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent3->GetHash()))); + BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_mixed_child->GetHash()))); + } +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp index 4ff326340d..538981ce36 100644 --- a/src/test/util/blockfilter.cpp +++ b/src/test/util/blockfilter.cpp @@ -8,6 +8,8 @@ #include <node/blockstorage.h> #include <validation.h> +using node::ReadBlockFromDisk; +using node::UndoReadFromDisk; bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) { diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index a9092bd0ef..09f96a033c 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -16,7 +16,7 @@ #include <boost/test/unit_test.hpp> -const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; +const auto NoMalleation = [](CAutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to @@ -24,7 +24,7 @@ const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; */ template<typename F = decltype(NoMalleation)> static bool -CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F malleation = NoMalleation) { // Write out a snapshot to the test's tempdir. // @@ -43,7 +43,7 @@ CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleati // FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; - SnapshotMetadata metadata; + node::SnapshotMetadata metadata; auto_infile >> metadata; malleation(auto_infile, metadata); diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 550cda892f..5ed8598e8e 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -16,6 +16,9 @@ #include <validation.h> #include <versionbits.h> +using node::BlockAssembler; +using node::NodeContext; + CTxIn generatetoaddress(const NodeContext& node, const std::string& address) { const auto dest = DecodeDestination(address); diff --git a/src/test/util/mining.h b/src/test/util/mining.h index e6cdbfd6d5..09e712cd35 100644 --- a/src/test/util/mining.h +++ b/src/test/util/mining.h @@ -13,18 +13,20 @@ class CBlock; class CChainParams; class CScript; class CTxIn; +namespace node { struct NodeContext; +} // namespace node /** Create a blockchain, starting from genesis */ std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const CChainParams& params); /** Returns the generated coin */ -CTxIn MineBlock(const NodeContext&, const CScript& coinbase_scriptPubKey); +CTxIn MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); /** Prepare a block to be mined */ -std::shared_ptr<CBlock> PrepareBlock(const NodeContext&, const CScript& coinbase_scriptPubKey); +std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); /** RPC-like helper function, returns the generated coin */ -CTxIn generatetoaddress(const NodeContext&, const std::string& address); +CTxIn generatetoaddress(const node::NodeContext&, const std::string& address); #endif // BITCOIN_TEST_UTIL_MINING_H diff --git a/src/test/util/net.h b/src/test/util/net.h index 006e876c1a..20c45058a1 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -13,6 +13,7 @@ #include <array> #include <cassert> #include <cstring> +#include <memory> #include <string> struct ConnmanTestMsg : public CConnman { @@ -126,6 +127,23 @@ public: int Connect(const sockaddr*, socklen_t) const override { return 0; } + std::unique_ptr<Sock> Accept(sockaddr* addr, socklen_t* addr_len) const override + { + if (addr != nullptr) { + // Pretend all connections come from 5.5.5.5:6789 + memset(addr, 0x00, *addr_len); + const socklen_t write_len = static_cast<socklen_t>(sizeof(sockaddr_in)); + if (*addr_len >= write_len) { + *addr_len = write_len; + sockaddr_in* addr_in = reinterpret_cast<sockaddr_in*>(addr); + addr_in->sin_family = AF_INET; + memset(&addr_in->sin_addr, 0x05, sizeof(addr_in->sin_addr)); + addr_in->sin_port = htons(6789); + } + } + return std::make_unique<StaticContentsSock>(""); + }; + int GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const override { std::memset(opt_val, 0x0, *opt_len); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 3a37aeb531..c968e4d124 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -42,6 +42,15 @@ #include <walletinitinterface.h> #include <functional> +#include <stdexcept> + +using node::BlockAssembler; +using node::CalculateCacheSizes; +using node::LoadChainstate; +using node::RegenerateCommitments; +using node::VerifyLoadedChainstate; +using node::fPruneMode; +using node::fReindex; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; UrlDecodeFn* const URL_DECODE = nullptr; @@ -80,7 +89,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve m_args{} { m_node.args = &gArgs; - const std::vector<const char*> arguments = Cat( + std::vector<const char*> arguments = Cat( { "dummy", "-printtoconsole=0", @@ -92,6 +101,9 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve "-debugexclude=leveldb", }, extra_args); + if (G_TEST_COMMAND_LINE_ARGUMENTS) { + arguments = Cat(arguments, G_TEST_COMMAND_LINE_ARGUMENTS()); + } util::ThreadRename("test"); fs::create_directories(m_path_root); m_args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); @@ -100,9 +112,10 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve { SetupServerArgs(*m_node.args); std::string error; - const bool success{m_node.args->ParseParameters(arguments.size(), arguments.data(), error)}; - assert(success); - assert(error.empty()); + if (!m_node.args->ParseParameters(arguments.size(), arguments.data(), error)) { + m_node.args->ClearArgs(); + throw std::runtime_error{error}; + } } SelectParams(chainName); SeedInsecureRand(); @@ -182,35 +195,37 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); - auto rv = LoadChainstate(fReindex.load(), - *Assert(m_node.chainman.get()), - Assert(m_node.mempool.get()), - fPruneMode, - chainparams.GetConsensus(), - m_args.GetBoolArg("-reindex-chainstate", false), - m_cache_sizes.block_tree_db, - m_cache_sizes.coins_db, - m_cache_sizes.coins, - true, - true); - assert(!rv.has_value()); - - auto maybe_verify_failure = VerifyLoadedChainstate( + auto maybe_load_error = LoadChainstate(fReindex.load(), + *Assert(m_node.chainman.get()), + Assert(m_node.mempool.get()), + fPruneMode, + chainparams.GetConsensus(), + m_args.GetBoolArg("-reindex-chainstate", false), + m_cache_sizes.block_tree_db, + m_cache_sizes.coins_db, + m_cache_sizes.coins, + /*block_tree_db_in_memory=*/true, + /*coins_db_in_memory=*/true); + assert(!maybe_load_error.has_value()); + + auto maybe_verify_error = VerifyLoadedChainstate( *Assert(m_node.chainman), fReindex.load(), m_args.GetBoolArg("-reindex-chainstate", false), chainparams.GetConsensus(), m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS), m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL), - static_cast<int64_t(*)()>(GetTime)); - assert(!maybe_verify_failure.has_value()); + /*get_unix_time_seconds=*/static_cast<int64_t(*)()>(GetTime)); + assert(!maybe_verify_error.has_value()); BlockValidationState state; if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<AddrMan>(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); + m_node.addrman = std::make_unique<AddrMan>(/*asmap=*/std::vector<bool>(), + /*deterministic=*/false, + m_node.args->GetIntArg("-checkaddrman", 0)); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 3fa75ad9ca..a1b7525cf4 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -19,18 +19,24 @@ #include <util/string.h> #include <util/vector.h> +#include <functional> #include <type_traits> #include <vector> /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; +/** Retrieve the command line arguments. */ +extern const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS; + // Enable BOOST_CHECK_EQUAL for enum class types +namespace std { template <typename T> std::ostream& operator<<(typename std::enable_if<std::is_enum<T>::value, std::ostream>::type& stream, const T& e) { return stream << static_cast<typename std::underlying_type<T>::type>(e); } +} // namespace std /** * This global and the helpers that use it are not thread-safe. @@ -76,7 +82,7 @@ static constexpr CAmount CENT{1000000}; */ struct BasicTestingSetup { ECCVerifyHandle globalVerifyHandle; - NodeContext m_node; + node::NodeContext m_node; explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); @@ -90,7 +96,7 @@ struct BasicTestingSetup { * initialization behaviour. */ struct ChainTestingSetup : public BasicTestingSetup { - CacheSizes m_cache_sizes{}; + node::CacheSizes m_cache_sizes{}; explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~ChainTestingSetup(); diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 4273dfe719..52aaeabccf 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -12,6 +12,8 @@ #include <wallet/wallet.h> #endif +using wallet::CWallet; + const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; #ifdef ENABLE_WALLET diff --git a/src/test/util/wallet.h b/src/test/util/wallet.h index 565ef1756a..31281bf70e 100644 --- a/src/test/util/wallet.h +++ b/src/test/util/wallet.h @@ -7,7 +7,9 @@ #include <string> +namespace wallet { class CWallet; +} // namespace wallet // Constants // @@ -16,9 +18,9 @@ extern const std::string ADDRESS_BCRT1_UNSPENDABLE; // RPC-like // /** Import the address to the wallet */ -void importaddress(CWallet& wallet, const std::string& address); +void importaddress(wallet::CWallet& wallet, const std::string& address); /** Returns a new address from the wallet */ -std::string getnewaddress(CWallet& w); +std::string getnewaddress(wallet::CWallet& w); #endif // BITCOIN_TEST_UTIL_WALLET_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 077d6e2d37..20d27a237d 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -15,6 +15,7 @@ #include <util/getuniquepath.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> +#include <util/overflow.h> #include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> @@ -23,6 +24,8 @@ #include <array> #include <optional> +#include <limits> +#include <map> #include <stdint.h> #include <string.h> #include <thread> @@ -1463,6 +1466,38 @@ BOOST_AUTO_TEST_CASE(test_IsDigit) BOOST_CHECK_EQUAL(IsDigit(9), false); } +/* Check for overflow */ +template <typename T> +static void TestAddMatrixOverflow() +{ + constexpr T MAXI{std::numeric_limits<T>::max()}; + BOOST_CHECK(!CheckedAdd(T{1}, MAXI)); + BOOST_CHECK(!CheckedAdd(MAXI, MAXI)); + BOOST_CHECK_EQUAL(0, CheckedAdd(T{0}, T{0}).value()); + BOOST_CHECK_EQUAL(MAXI, CheckedAdd(T{0}, MAXI).value()); + BOOST_CHECK_EQUAL(MAXI, CheckedAdd(T{1}, MAXI - 1).value()); +} + +/* Check for overflow or underflow */ +template <typename T> +static void TestAddMatrix() +{ + TestAddMatrixOverflow<T>(); + constexpr T MINI{std::numeric_limits<T>::min()}; + constexpr T MAXI{std::numeric_limits<T>::max()}; + BOOST_CHECK(!CheckedAdd(T{-1}, MINI)); + BOOST_CHECK(!CheckedAdd(MINI, MINI)); + BOOST_CHECK_EQUAL(MINI, CheckedAdd(T{0}, MINI).value()); + BOOST_CHECK_EQUAL(MINI, CheckedAdd(T{-1}, MINI + 1).value()); + BOOST_CHECK_EQUAL(-1, CheckedAdd(MINI, MAXI).value()); +} + +BOOST_AUTO_TEST_CASE(util_overflow) +{ + TestAddMatrixOverflow<unsigned>(); + TestAddMatrix<signed>(); +} + BOOST_AUTO_TEST_CASE(test_ParseInt32) { int32_t n; @@ -1588,6 +1623,11 @@ BOOST_AUTO_TEST_CASE(test_ToIntegral) BOOST_CHECK(!ToIntegral<uint8_t>("256")); } +int64_t atoi64_legacy(const std::string& str) +{ + return strtoll(str.c_str(), nullptr, 10); +} + BOOST_AUTO_TEST_CASE(test_LocaleIndependentAtoi) { BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1234"), 1'234); @@ -1615,48 +1655,68 @@ BOOST_AUTO_TEST_CASE(test_LocaleIndependentAtoi) BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(""), 0); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("aap"), 0); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0x1"), 0); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-32482348723847471234"), 0); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("32482348723847471234"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-32482348723847471234"), -2'147'483'647 - 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("32482348723847471234"), 2'147'483'647); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775809"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775809"), -9'223'372'036'854'775'807LL - 1LL); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775808"), -9'223'372'036'854'775'807LL - 1LL); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775807"), 9'223'372'036'854'775'807); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775808"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775808"), 9'223'372'036'854'775'807); + + std::map<std::string, int64_t> atoi64_test_pairs = { + {"-9223372036854775809", std::numeric_limits<int64_t>::min()}, + {"-9223372036854775808", -9'223'372'036'854'775'807LL - 1LL}, + {"9223372036854775807", 9'223'372'036'854'775'807}, + {"9223372036854775808", std::numeric_limits<int64_t>::max()}, + {"+-", 0}, + {"0x1", 0}, + {"ox1", 0}, + {"", 0}, + }; + + for (const auto& pair : atoi64_test_pairs) { + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>(pair.first), pair.second); + } + + // Ensure legacy compatibility with previous versions of Bitcoin Core's atoi64 + for (const auto& pair : atoi64_test_pairs) { + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>(pair.first), atoi64_legacy(pair.first)); + } BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("-1"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("0"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551615"), 18'446'744'073'709'551'615ULL); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551616"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551616"), 18'446'744'073'709'551'615ULL); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483649"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483649"), -2'147'483'648LL); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483648"), -2'147'483'648LL); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483647"), 2'147'483'647); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483648"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483648"), 2'147'483'647); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("-1"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("0"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967295"), 4'294'967'295U); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967296"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967296"), 4'294'967'295U); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32769"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32769"), -32'768); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32768"), -32'768); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32767"), 32'767); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32768"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32768"), 32'767); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("-1"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("0"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65535"), 65'535U); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65536"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65536"), 65'535U); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-129"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-129"), -128); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-128"), -128); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("127"), 127); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("128"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("128"), 127); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("-1"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("0"), 0U); BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("255"), 255U); - BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("256"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("256"), 255U); } BOOST_AUTO_TEST_CASE(test_ParseInt64) diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 3f9c28fce0..c5b1dabcb7 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -19,6 +19,8 @@ #include <thread> +using node::BlockAssembler; + namespace validation_block_tests { struct MinerTestingSetup : public RegTestingSetup { std::shared_ptr<CBlock> Block(const uint256& prev_hash); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 4be568ab47..1beef5cf04 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -43,6 +43,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) c1.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); + BOOST_REQUIRE(c1.LoadGenesisBlock()); // Need at least one block loaded to be able to flush caches // Add a coin to the in-memory cache, upsize once, then downsize. { diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 5b36f261e6..f5742b65a1 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -20,6 +20,8 @@ #include <boost/test/unit_test.hpp> +using node::SnapshotMetadata; + BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) //! Basic tests for ChainstateManager. diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 23d0eacb4b..a34895d4ae 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -9,6 +9,8 @@ #include <boost/test/unit_test.hpp> +using node::BlockManager; + BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, ChainTestingSetup) //! Test utilities for detecting when we need to flush the coins cache based diff --git a/src/txmempool.cpp b/src/txmempool.cpp index dc2769b81e..fb5652d0a0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -116,10 +116,9 @@ size_t CTxMemPoolEntry::GetTxSize() const return GetVirtualTransactionSize(nTxWeight, sigOpCost); } -// Update the given tx for any in-mempool descendants. -// Assumes that CTxMemPool::m_children is correct for the given tx and all -// descendants. -void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set<uint256> &setExclude) +void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendants, + const std::set<uint256>& setExclude, std::set<uint256>& descendants_to_remove, + uint64_t ancestor_size_limit, uint64_t ancestor_count_limit) { CTxMemPoolEntry::Children stageEntries, descendants; stageEntries = updateIt->GetMemPoolChildrenConst(); @@ -156,17 +155,18 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendan cachedDescendants[updateIt].insert(mapTx.iterator_to(descendant)); // Update ancestor state for each descendant mapTx.modify(mapTx.iterator_to(descendant), update_ancestor_state(updateIt->GetTxSize(), updateIt->GetModifiedFee(), 1, updateIt->GetSigOpCost())); + // Don't directly remove the transaction here -- doing so would + // invalidate iterators in cachedDescendants. Mark it for removal + // by inserting into descendants_to_remove. + if (descendant.GetCountWithAncestors() > ancestor_count_limit || descendant.GetSizeWithAncestors() > ancestor_size_limit) { + descendants_to_remove.insert(descendant.GetTx().GetHash()); + } } } mapTx.modify(updateIt, update_descendant_state(modifySize, modifyFee, modifyCount)); } -// vHashesToUpdate is the set of transaction hashes from a disconnected block -// which has been re-added to the mempool. -// for each entry, look for descendants that are outside vHashesToUpdate, and -// add fee/size information for such descendants to the parent. -// for each such descendant, also update the ancestor state to include the parent. -void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashesToUpdate) +void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashesToUpdate, uint64_t ancestor_size_limit, uint64_t ancestor_count_limit) { AssertLockHeld(cs); // For each entry in vHashesToUpdate, store the set of in-mempool, but not @@ -178,6 +178,8 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes // accounted for in the state of their ancestors) std::set<uint256> setAlreadyIncluded(vHashesToUpdate.begin(), vHashesToUpdate.end()); + std::set<uint256> descendants_to_remove; + // Iterate in reverse, so that whenever we are looking at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that @@ -207,7 +209,15 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes } } } // release epoch guard for UpdateForDescendants - UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded); + UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded, descendants_to_remove, ancestor_size_limit, ancestor_count_limit); + } + + for (const auto& txid : descendants_to_remove) { + // This txid may have been removed already in a prior call to removeRecursive. + // Therefore we ensure it is not yet removed already. + if (const std::optional<txiter> txiter = GetIter(txid)) { + removeRecursive((*txiter)->GetTx(), MemPoolRemovalReason::SIZELIMIT); + } } } diff --git a/src/txmempool.h b/src/txmempool.h index b8c508fd90..e7e5a3c402 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -312,16 +312,6 @@ public: } }; -struct update_lock_points -{ - explicit update_lock_points(const LockPoints& _lp) : lp(_lp) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } - -private: - const LockPoints& lp; -}; - // Multi_index tag names struct descendant_score {}; struct entry_time {}; @@ -599,10 +589,14 @@ public: void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors, bool validFeeEstimate = true) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); - /** After reorg, check if mempool entries are now non-final, premature coinbase spends, or have - * invalid lockpoints. Update lockpoints and remove entries (and descendants of entries) that - * are no longer valid. */ - void removeForReorg(CChain& chain, std::function<bool(txiter)> check_final_and_mature) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); + /** After reorg, filter the entries that would no longer be valid in the next block, and update + * the entries' cached LockPoints if needed. The mempool does not have any knowledge of + * consensus rules. It just appplies the callable function and removes the ones for which it + * returns true. + * @param[in] filter_final_and_mature Predicate that checks the relevant validation rules + * and updates an entry's LockPoints. + * */ + void removeForReorg(CChain& chain, std::function<bool(txiter)> filter_final_and_mature) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeConflicts(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(cs); void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -642,16 +636,25 @@ public: */ void RemoveStaged(setEntries& stage, bool updateDescendants, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); - /** When adding transactions from a disconnected block back to the mempool, - * new mempool entries may have children in the mempool (which is generally - * not the case when otherwise adding transactions). - * UpdateTransactionsFromBlock() will find child transactions and update the - * descendant state for each transaction in vHashesToUpdate (excluding any - * child transactions present in vHashesToUpdate, which are already accounted - * for). Note: vHashesToUpdate should be the set of transactions from the - * disconnected block that have been accepted back into the mempool. + /** UpdateTransactionsFromBlock is called when adding transactions from a + * disconnected block back to the mempool, new mempool entries may have + * children in the mempool (which is generally not the case when otherwise + * adding transactions). + * @post updated descendant state for descendants of each transaction in + * vHashesToUpdate (excluding any child transactions present in + * vHashesToUpdate, which are already accounted for). Updated state + * includes add fee/size information for such descendants to the + * parent and updated ancestor state to include the parent. + * + * @param[in] vHashesToUpdate The set of txids from the + * disconnected block that have been accepted back into the mempool. + * @param[in] ancestor_size_limit The maximum allowed size in virtual + * bytes of an entry and its ancestors + * @param[in] ancestor_count_limit The maximum allowed number of + * transactions including the entry and its ancestors. */ - void UpdateTransactionsFromBlock(const std::vector<uint256>& vHashesToUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main) LOCKS_EXCLUDED(m_epoch); + void UpdateTransactionsFromBlock(const std::vector<uint256>& vHashesToUpdate, + uint64_t ancestor_size_limit, uint64_t ancestor_count_limit) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main) LOCKS_EXCLUDED(m_epoch); /** Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) @@ -800,19 +803,38 @@ private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the * mempool but may have child transactions in the mempool, eg during a - * chain reorg. setExclude is the set of descendant transactions in the - * mempool that must not be accounted for (because any descendants in - * setExclude were added to the mempool after the transaction being - * updated and hence their state is already reflected in the parent - * state). + * chain reorg. + * + * @pre CTxMemPool::m_children is correct for the given tx and all + * descendants. + * @pre cachedDescendants is an accurate cache where each entry has all + * descendants of the corresponding key, including those that should + * be removed for violation of ancestor limits. + * @post if updateIt has any non-excluded descendants, cachedDescendants has + * a new cache line for updateIt. + * @post descendants_to_remove has a new entry for any descendant which exceeded + * ancestor limits relative to updateIt. * - * cachedDescendants will be updated with the descendants of the transaction - * being updated, so that future invocations don't need to walk the - * same transaction again, if encountered in another transaction chain. + * @param[in] updateIt the entry to update for its descendants + * @param[in,out] cachedDescendants a cache where each line corresponds to all + * descendants. It will be updated with the descendants of the transaction + * being updated, so that future invocations don't need to walk the same + * transaction again, if encountered in another transaction chain. + * @param[in] setExclude the set of descendant transactions in the mempool + * that must not be accounted for (because any descendants in setExclude + * were added to the mempool after the transaction being updated and hence + * their state is already reflected in the parent state). + * @param[out] descendants_to_remove Populated with the txids of entries that + * exceed ancestor limits. It's the responsibility of the caller to + * removeRecursive them. + * @param[in] ancestor_size_limit the max number of ancestral bytes allowed + * for any descendant + * @param[in] ancestor_count_limit the max number of ancestor transactions + * allowed for any descendant */ - void UpdateForDescendants(txiter updateIt, - cacheMap &cachedDescendants, - const std::set<uint256> &setExclude) EXCLUSIVE_LOCKS_REQUIRED(cs); + void UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendants, + const std::set<uint256>& setExclude, std::set<uint256>& descendants_to_remove, + uint64_t ancestor_size_limit, uint64_t ancestor_count_limit) EXCLUSIVE_LOCKS_REQUIRED(cs); /** Update ancestors of hash to add/remove it as a descendant transaction. */ void UpdateAncestorsOf(bool add, txiter hash, setEntries &setAncestors) EXCLUSIVE_LOCKS_REQUIRED(cs); /** Set ancestor state for an entry */ diff --git a/src/util/fastrange.h b/src/util/fastrange.h new file mode 100644 index 0000000000..77cb883ce0 --- /dev/null +++ b/src/util/fastrange.h @@ -0,0 +1,51 @@ +// Copyright (c) 2018-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_FASTRANGE_H +#define BITCOIN_UTIL_FASTRANGE_H + +#include <cstdint> + +/* This file offers implementations of the fast range reduction technique described + * in https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + * + * In short, they take an integer x and a range n, and return the upper bits of + * (x * n). If x is uniformly distributed over its domain, the result is as close to + * uniformly distributed over [0, n) as (x mod n) would be, but significantly faster. + */ + +/** Fast range reduction with 32-bit input and 32-bit range. */ +static inline uint32_t FastRange32(uint32_t x, uint32_t n) +{ + return (uint64_t{x} * n) >> 32; +} + +/** Fast range reduction with 64-bit input and 64-bit range. */ +static inline uint64_t FastRange64(uint64_t x, uint64_t n) +{ +#ifdef __SIZEOF_INT128__ + return (static_cast<unsigned __int128>(x) * static_cast<unsigned __int128>(n)) >> 64; +#else + // To perform the calculation on 64-bit numbers without losing the + // result to overflow, split the numbers into the most significant and + // least significant 32 bits and perform multiplication piece-wise. + // + // See: https://stackoverflow.com/a/26855440 + const uint64_t x_hi = x >> 32; + const uint64_t x_lo = x & 0xFFFFFFFF; + const uint64_t n_hi = n >> 32; + const uint64_t n_lo = n & 0xFFFFFFFF; + + const uint64_t ac = x_hi * n_hi; + const uint64_t ad = x_hi * n_lo; + const uint64_t bc = x_lo * n_hi; + const uint64_t bd = x_lo * n_lo; + + const uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); + const uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); + return upper64; +#endif +} + +#endif // BITCOIN_UTIL_FASTRANGE_H diff --git a/src/util/golombrice.h b/src/util/golombrice.h index 67d262406f..4ff4f6d7e5 100644 --- a/src/util/golombrice.h +++ b/src/util/golombrice.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_UTIL_GOLOMBRICE_H #define BITCOIN_UTIL_GOLOMBRICE_H +#include <util/fastrange.h> + #include <streams.h> #include <cstdint> diff --git a/src/util/overflow.h b/src/util/overflow.h new file mode 100644 index 0000000000..5982af8d04 --- /dev/null +++ b/src/util/overflow.h @@ -0,0 +1,31 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_OVERFLOW_H +#define BITCOIN_UTIL_OVERFLOW_H + +#include <limits> +#include <type_traits> + +template <class T> +[[nodiscard]] bool AdditionOverflow(const T i, const T j) noexcept +{ + static_assert(std::is_integral<T>::value, "Integral required."); + if (std::numeric_limits<T>::is_signed) { + return (i > 0 && j > std::numeric_limits<T>::max() - i) || + (i < 0 && j < std::numeric_limits<T>::min() - i); + } + return std::numeric_limits<T>::max() - i < j; +} + +template <class T> +[[nodiscard]] std::optional<T> CheckedAdd(const T i, const T j) noexcept +{ + if (AdditionOverflow(i, j)) { + return std::nullopt; + } + return i + j; +} + +#endif // BITCOIN_UTIL_OVERFLOW_H diff --git a/src/util/settings.cpp b/src/util/settings.cpp index 442a55ffb0..683b7ae652 100644 --- a/src/util/settings.cpp +++ b/src/util/settings.cpp @@ -112,7 +112,7 @@ bool WriteSettings(const fs::path& path, errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", fs::PathToString(path))); return false; } - file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl; + file << out.write(/* prettyIndent= */ 4, /* indentLevel= */ 1) << std::endl; file.close(); return true; } diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 1a4d67a65e..2029d70a37 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -10,6 +10,7 @@ #include <util/system.h> #include <util/time.h> +#include <memory> #include <stdexcept> #include <string> @@ -73,6 +74,32 @@ int Sock::Connect(const sockaddr* addr, socklen_t addr_len) const return connect(m_socket, addr, addr_len); } +std::unique_ptr<Sock> Sock::Accept(sockaddr* addr, socklen_t* addr_len) const +{ +#ifdef WIN32 + static constexpr auto ERR = INVALID_SOCKET; +#else + static constexpr auto ERR = SOCKET_ERROR; +#endif + + std::unique_ptr<Sock> sock; + + const auto socket = accept(m_socket, addr, addr_len); + if (socket != ERR) { + try { + sock = std::make_unique<Sock>(socket); + } catch (const std::exception&) { +#ifdef WIN32 + closesocket(socket); +#else + close(socket); +#endif + } + } + + return sock; +} + int Sock::GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const { return getsockopt(m_socket, level, opt_name, static_cast<char*>(opt_val), opt_len); diff --git a/src/util/sock.h b/src/util/sock.h index 59cc8c0b1d..7510482857 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -10,6 +10,7 @@ #include <util/time.h> #include <chrono> +#include <memory> #include <string> /** @@ -97,6 +98,14 @@ public: [[nodiscard]] virtual int Connect(const sockaddr* addr, socklen_t addr_len) const; /** + * accept(2) wrapper. Equivalent to `std::make_unique<Sock>(accept(this->Get(), addr, addr_len))`. + * Code that uses this wrapper can be unit tested if this method is overridden by a mock Sock + * implementation. + * The returned unique_ptr is empty if `accept()` failed in which case errno will be set. + */ + [[nodiscard]] virtual std::unique_ptr<Sock> Accept(sockaddr* addr, socklen_t* addr_len) const; + + /** * getsockopt(2) wrapper. Equivalent to * `getsockopt(this->Get(), level, opt_name, opt_val, opt_len)`. Code that uses this * wrapper can be unit tested if this method is overridden by a mock Sock implementation. diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 0945556b5e..1f83fa3ffa 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -16,6 +16,7 @@ #include <charconv> #include <cstdint> #include <iterator> +#include <limits> #include <optional> #include <string> #include <vector> @@ -93,8 +94,12 @@ void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); // New code should use ToIntegral or the ParseInt* functions // which provide parse error feedback. // -// The goal of LocaleIndependentAtoi is to replicate the exact defined behaviour -// of atoi and atoi64 as they behave under the "C" locale. +// The goal of LocaleIndependentAtoi is to replicate the defined behaviour of +// std::atoi as it behaves under the "C" locale, and remove some undefined +// behavior. If the parsed value is bigger than the integer type's maximum +// value, or smaller than the integer type's minimum value, std::atoi has +// undefined behavior, while this function returns the maximum or minimum +// values, respectively. template <typename T> T LocaleIndependentAtoi(const std::string& str) { @@ -109,7 +114,15 @@ T LocaleIndependentAtoi(const std::string& str) s = s.substr(1); } auto [_, error_condition] = std::from_chars(s.data(), s.data() + s.size(), result); - if (error_condition != std::errc{}) { + if (error_condition == std::errc::result_out_of_range) { + if (s.length() >= 1 && s[0] == '-') { + // Saturate underflow, per strtoll's behavior. + return std::numeric_limits<T>::min(); + } else { + // Saturate overflow, per strtoll's behavior. + return std::numeric_limits<T>::max(); + } + } else if (error_condition != std::errc{}) { return 0; } return result; diff --git a/src/util/system.cpp b/src/util/system.cpp index 8f35b7b6c6..e34cdc7fb9 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -6,11 +6,6 @@ #include <util/system.h> #ifdef ENABLE_EXTERNAL_SIGNER -#if defined(WIN32) && !defined(__kernel_entry) -// A workaround for boost 1.71 incompatibility with mingw-w64 compiler. -// For details see https://github.com/bitcoin/bitcoin/pull/22348. -#define __kernel_entry -#endif #include <boost/process.hpp> #endif // ENABLE_EXTERNAL_SIGNER diff --git a/src/validation.cpp b/src/validation.cpp index a98ffe006d..1b1d01a4c2 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -62,6 +62,25 @@ #include <boost/algorithm/string/replace.hpp> +using node::BLOCKFILE_CHUNK_SIZE; +using node::BlockManager; +using node::BlockMap; +using node::CBlockIndexWorkComparator; +using node::CCoinsStats; +using node::CoinStatsHashType; +using node::GetUTXOStats; +using node::OpenBlockFile; +using node::ReadBlockFromDisk; +using node::SnapshotMetadata; +using node::UNDOFILE_CHUNK_SIZE; +using node::UndoReadFromDisk; +using node::UnlinkPrunedFiles; +using node::fHavePruned; +using node::fImporting; +using node::fPruneMode; +using node::fReindex; +using node::nPruneTarget; + #define MICRO 0.000001 #define MILLI 0.001 @@ -133,16 +152,6 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -// Internal stuff from blockstorage ... -extern RecursiveMutex cs_LastBlockFile; -extern std::vector<CBlockFileInfo> vinfoBlockFile; -extern int nLastBlockFile; -extern bool fCheckForPruning; -extern std::set<CBlockIndex*> setDirtyBlockIndex; -extern std::set<int> setDirtyFileInfo; -void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); -// ... TODO move fully to blockstorage - CBlockIndex* CChainState::FindForkInGlobalIndex(const CBlockLocator& locator) const { AssertLockHeld(cs_main); @@ -338,43 +347,59 @@ void CChainState::MaybeUpdateMempoolForReorg( // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in // the disconnectpool that were added back and cleans up the mempool state. - m_mempool->UpdateTransactionsFromBlock(vHashUpdate); - - const auto check_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it) + const uint64_t ancestor_count_limit = gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + const uint64_t ancestor_size_limit = gArgs.GetIntArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; + m_mempool->UpdateTransactionsFromBlock(vHashUpdate, ancestor_size_limit, ancestor_count_limit); + + // Predicate to use for filtering transactions in removeForReorg. + // Checks whether the transaction is still final and, if it spends a coinbase output, mature. + // Also updates valid entries' cached LockPoints if needed. + // If false, the tx is still valid and its lockpoints are updated. + // If true, the tx would be invalid in the next block; remove this entry and all of its descendants. + const auto filter_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it) EXCLUSIVE_LOCKS_REQUIRED(m_mempool->cs, ::cs_main) { - bool should_remove = false; AssertLockHeld(m_mempool->cs); AssertLockHeld(::cs_main); const CTransaction& tx = it->GetTx(); + + // The transaction must be final. + if (!CheckFinalTx(m_chain.Tip(), tx, flags)) return true; LockPoints lp = it->GetLockPoints(); const bool validLP{TestLockPointValidity(m_chain, lp)}; CCoinsViewMemPool view_mempool(&CoinsTip(), *m_mempool); - if (!CheckFinalTx(m_chain.Tip(), tx, flags) - || !CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) { - // Note if CheckSequenceLocks fails the LockPoints may still be invalid - // So it's critical that we remove the tx and not depend on the LockPoints. - should_remove = true; - } else if (it->GetSpendsCoinbase()) { + // CheckSequenceLocks checks if the transaction will be final in the next block to be + // created on top of the new chain. We use useExistingLockPoints=false so that, instead of + // using the information in lp (which might now refer to a block that no longer exists in + // the chain), it will update lp to contain LockPoints relevant to the new chain. + if (!CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) { + // If CheckSequenceLocks fails, remove the tx and don't depend on the LockPoints. + return true; + } else if (!validLP) { + // If CheckSequenceLocks succeeded, it also updated the LockPoints. + // Now update the mempool entry lockpoints as well. + m_mempool->mapTx.modify(it, [&lp](CTxMemPoolEntry& e) { e.UpdateLockPoints(lp); }); + } + + // If the transaction spends any coinbase outputs, it must be mature. + if (it->GetSpendsCoinbase()) { for (const CTxIn& txin : tx.vin) { auto it2 = m_mempool->mapTx.find(txin.prevout.hash); if (it2 != m_mempool->mapTx.end()) continue; - const Coin &coin = CoinsTip().AccessCoin(txin.prevout); + const Coin& coin{CoinsTip().AccessCoin(txin.prevout)}; assert(!coin.IsSpent()); const auto mempool_spend_height{m_chain.Tip()->nHeight + 1}; - if (coin.IsSpent() || (coin.IsCoinBase() && mempool_spend_height - coin.nHeight < COINBASE_MATURITY)) { - should_remove = true; - break; + if (coin.IsCoinBase() && mempool_spend_height - coin.nHeight < COINBASE_MATURITY) { + return true; } } } - // CheckSequenceLocks updates lp. Update the mempool entry LockPoints. - if (!validLP) m_mempool->mapTx.modify(it, update_lock_points(lp)); - return should_remove; + // Transaction is still valid and cached LockPoints are updated. + return false; }; // We also need to remove any now-immature transactions - m_mempool->removeForReorg(m_chain, check_final_and_mature); + m_mempool->removeForReorg(m_chain, filter_final_and_mature); // Re-limit mempool size, in case we added any transactions LimitMempoolSize( *m_mempool, @@ -596,10 +621,10 @@ private: // Submit all transactions to the mempool and call ConsensusScriptChecks to add to the script // cache - should only be called after successful validation of all transactions in the package. - // The package may end up partially-submitted after size limitting; returns true if all + // The package may end up partially-submitted after size limiting; returns true if all // transactions are successfully added to the mempool, false otherwise. - bool FinalizePackage(const ATMPArgs& args, std::vector<Workspace>& workspaces, PackageValidationState& package_state, - std::map<const uint256, const MempoolAcceptResult>& results) + bool SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& workspaces, PackageValidationState& package_state, + std::map<const uint256, const MempoolAcceptResult>& results) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Compare a package's feerate against minimum allowed. @@ -1038,12 +1063,17 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) return true; } -bool MemPoolAccept::FinalizePackage(const ATMPArgs& args, std::vector<Workspace>& workspaces, - PackageValidationState& package_state, - std::map<const uint256, const MempoolAcceptResult>& results) +bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& workspaces, + PackageValidationState& package_state, + std::map<const uint256, const MempoolAcceptResult>& results) { AssertLockHeld(cs_main); AssertLockHeld(m_pool.cs); + // Sanity check: none of the transactions should be in the mempool, and none of the transactions + // should have a same-txid-different-witness equivalent in the mempool. + assert(std::all_of(workspaces.cbegin(), workspaces.cend(), [this](const auto& ws){ + return !m_pool.exists(GenTxid::Txid(ws.m_ptx->GetHash())); })); + bool all_submitted = true; // ConsensusScriptChecks adds to the script cache and is therefore consensus-critical; // CheckInputsFromMempoolAndCache asserts that transactions only spend coins available from the @@ -1053,18 +1083,24 @@ bool MemPoolAccept::FinalizePackage(const ATMPArgs& args, std::vector<Workspace> if (!ConsensusScriptChecks(args, ws)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since PolicyScriptChecks() passed, this should never fail. - all_submitted = Assume(false); + all_submitted = false; + package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, + strprintf("BUG! PolicyScriptChecks succeeded but ConsensusScriptChecks failed: %s", + ws.m_ptx->GetHash().ToString())); } // Re-calculate mempool ancestors to call addUnchecked(). They may have changed since the // last calculation done in PreChecks, since package ancestors have already been submitted. - std::string err_string; + std::string unused_err_string; if(!m_pool.CalculateMemPoolAncestors(*ws.m_entry, ws.m_ancestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, - m_limit_descendant_size, err_string)) { + m_limit_descendant_size, unused_err_string)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since PreChecks() and PackageMempoolChecks() both enforce limits, this should never fail. - all_submitted = Assume(false); + all_submitted = false; + package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, + strprintf("BUG! Mempool ancestors or descendants were underestimated: %s", + ws.m_ptx->GetHash().ToString())); } // If we call LimitMempoolSize() for each individual Finalize(), the mempool will not take // the transaction's descendant feerate into account because it hasn't seen them yet. Also, @@ -1074,7 +1110,9 @@ bool MemPoolAccept::FinalizePackage(const ATMPArgs& args, std::vector<Workspace> if (!Finalize(args, ws)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since LimitMempoolSize() won't be called, this should never fail. - all_submitted = Assume(false); + all_submitted = false; + package_state.Invalid(PackageValidationResult::PCKG_MEMPOOL_ERROR, + strprintf("BUG! Adding to mempool failed: %s", ws.m_ptx->GetHash().ToString())); } } @@ -1083,7 +1121,6 @@ bool MemPoolAccept::FinalizePackage(const ATMPArgs& args, std::vector<Workspace> LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); - if (!all_submitted) return false; // Find the wtxids of the transactions that made it into the mempool. Allow partial submission, // but don't report success unless they all made it into the mempool. @@ -1188,8 +1225,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: if (args.m_test_accept) return PackageMempoolAcceptResult(package_state, std::move(results)); - if (!FinalizePackage(args, workspaces, package_state, results)) { - package_state.Invalid(PackageValidationResult::PCKG_TX, "submission failed"); + if (!SubmitPackage(args, workspaces, package_state, results)) { + // PackageValidationState filled in by SubmitPackage(). return PackageMempoolAcceptResult(package_state, std::move(results)); } @@ -1214,11 +1251,13 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, return PackageMempoolAcceptResult(package_state, {}); } - const auto& child = package[package.size() - 1]; + // IsChildWithParents() guarantees the package is > 1 transactions. + assert(package.size() > 1); // The package must be 1 child with all of its unconfirmed parents. The package is expected to // be sorted, so the last transaction is the child. + const auto& child = package.back(); std::unordered_set<uint256, SaltedTxidHasher> unconfirmed_parent_txids; - std::transform(package.cbegin(), package.end() - 1, + std::transform(package.cbegin(), package.cend() - 1, std::inserter(unconfirmed_parent_txids, unconfirmed_parent_txids.end()), [](const auto& tx) { return tx->GetHash(); }); @@ -1250,10 +1289,14 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, LOCK(m_pool.cs); std::map<const uint256, const MempoolAcceptResult> results; - // As node operators are free to set their mempool policies however they please, it's possible - // for package transaction(s) to already be in the mempool, and we don't want to reject the - // entire package in that case (as that could be a censorship vector). Filter the transactions - // that are already in mempool and add their information to results, since we already have them. + // Node operators are free to set their mempool policies however they please, nodes may receive + // transactions in different orders, and malicious counterparties may try to take advantage of + // policy differences to pin or delay propagation of transactions. As such, it's possible for + // some package transaction(s) to already be in the mempool, and we don't want to reject the + // entire package in that case (as that could be a censorship vector). De-duplicate the + // transactions that are already in the mempool, and only call AcceptMultipleTransactions() with + // the new transactions. This ensures we don't double-count transaction counts and sizes when + // checking ancestor/descendant limits, or double-count transaction fees for fee-related policy. std::vector<CTransactionRef> txns_new; for (const auto& tx : package) { const auto& wtxid = tx->GetWitnessHash(); @@ -1274,9 +1317,10 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transaction for the mempool one. Note that we are ignoring the validity of the // package transaction passed in. // TODO: allow witness replacement in packages. - auto iter = m_pool.GetIter(wtxid); + auto iter = m_pool.GetIter(txid); assert(iter != std::nullopt); - results.emplace(txid, MempoolAcceptResult::MempoolTx(iter.value()->GetTxSize(), iter.value()->GetFee())); + // Provide the wtxid of the mempool tx so that the caller can look it up in the mempool. + results.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(iter.value()->GetTx().GetWitnessHash())); } else { // Transaction does not already exist in the mempool. txns_new.push_back(tx); @@ -1343,12 +1387,12 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx }(); // Uncache coins pertaining to transactions that were not submitted to the mempool. - // Ensure the coins cache is still within limits. if (test_accept || result.m_state.IsInvalid()) { for (const COutPoint& hashTx : coins_to_uncache) { active_chainstate.CoinsTip().Uncache(hashTx); } } + // Ensure the coins cache is still within limits. BlockValidationState state_dummy; active_chainstate.FlushStateToDisk(state_dummy, FlushStateMode::PERIODIC); return result; @@ -1505,7 +1549,7 @@ void CChainState::InvalidBlockFound(CBlockIndex* pindex, const BlockValidationSt if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; m_chainman.m_failed_blocks.insert(pindex); - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); } @@ -2135,13 +2179,13 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, if (fJustCheck) return true; - if (!WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { + if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { return false; } if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); } assert(pindex->phashBlock); @@ -2214,8 +2258,8 @@ bool CChainState::FlushStateToDisk( bool fDoFullFlush = false; CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); - LOCK(cs_LastBlockFile); - if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { + LOCK(m_blockman.cs_LastBlockFile); + if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { // make sure we don't prune above the blockfilterindexes bestblocks // pruning is height-based int last_prune = m_chain.Height(); // last height we can prune @@ -2231,7 +2275,7 @@ bool CChainState::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); - fCheckForPruning = false; + m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { fFlushForPrune = true; @@ -2269,26 +2313,14 @@ bool CChainState::FlushStateToDisk( LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); + m_blockman.FlushBlockFile(); } // Then update all block file information (which may refer to block and undo files). { LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCH); - std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; - vFiles.reserve(setDirtyFileInfo.size()); - for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { - vFiles.push_back(std::make_pair(*it, &vinfoBlockFile[*it])); - setDirtyFileInfo.erase(it++); - } - std::vector<const CBlockIndex*> vBlocks; - vBlocks.reserve(setDirtyBlockIndex.size()); - for (std::set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { - vBlocks.push_back(*it); - setDirtyBlockIndex.erase(it++); - } - if (!m_blockman.m_block_tree_db->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + if (!m_blockman.WriteBlockIndexDB()) { return AbortNode(state, "Failed to write to block index database"); } } @@ -2348,7 +2380,7 @@ void CChainState::ForceFlushStateToDisk() void CChainState::PruneAndFlush() { BlockValidationState state; - fCheckForPruning = true; + m_blockman.m_check_for_pruning = true; if (!this->FlushStateToDisk(state, FlushStateMode::NONE)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); } @@ -3018,14 +3050,14 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // are no blocks that meet the "have data and are not invalid per // nStatus" criteria for inclusion in setBlockIndexCandidates). invalid_walk_tip->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(invalid_walk_tip); + m_blockman.m_dirty_blockindex.insert(invalid_walk_tip); setBlockIndexCandidates.erase(invalid_walk_tip); setBlockIndexCandidates.insert(invalid_walk_tip->pprev); if (invalid_walk_tip->pprev == to_mark_failed && (to_mark_failed->nStatus & BLOCK_FAILED_VALID)) { // We only want to mark the last disconnected block as BLOCK_FAILED_VALID; its children // need to be BLOCK_FAILED_CHILD instead. to_mark_failed->nStatus = (to_mark_failed->nStatus ^ BLOCK_FAILED_VALID) | BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(to_mark_failed); + m_blockman.m_dirty_blockindex.insert(to_mark_failed); } // Add any equal or more work headers to setBlockIndexCandidates @@ -3055,7 +3087,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // Mark pindex (or the last disconnected block) as invalid, even when it never was in the main chain to_mark_failed->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(to_mark_failed); + m_blockman.m_dirty_blockindex.insert(to_mark_failed); setBlockIndexCandidates.erase(to_mark_failed); m_chainman.m_failed_blocks.insert(to_mark_failed); @@ -3094,7 +3126,7 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { while (it != m_blockman.m_block_index.end()) { if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { it->second->nStatus &= ~BLOCK_FAILED_MASK; - setDirtyBlockIndex.insert(it->second); + m_blockman.m_dirty_blockindex.insert(it->second); if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), it->second)) { setBlockIndexCandidates.insert(it->second); } @@ -3111,7 +3143,7 @@ void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { while (pindex != nullptr) { if (pindex->nStatus & BLOCK_FAILED_MASK) { pindex->nStatus &= ~BLOCK_FAILED_MASK; - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); m_chainman.m_failed_blocks.erase(pindex); } pindex = pindex->pprev; @@ -3131,7 +3163,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); - setDirtyBlockIndex.insert(pindexNew); + m_blockman.m_dirty_blockindex.insert(pindexNew); if (pindexNew->pprev == nullptr || pindexNew->pprev->HaveTxsDownloaded()) { // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. @@ -3493,7 +3525,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida CBlockIndex* invalid_walk = pindexPrev; while (invalid_walk != failedit) { invalid_walk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(invalid_walk); + m_blockman.m_dirty_blockindex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); @@ -3531,7 +3563,10 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& } if (NotifyHeaderTip(ActiveChainstate())) { if (ActiveChainstate().IsInitialBlockDownload() && ppindex && *ppindex) { - LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", (*ppindex)->nHeight, 100.0/((*ppindex)->nHeight+(GetAdjustedTime() - (*ppindex)->GetBlockTime()) / Params().GetConsensus().nPowTargetSpacing) * (*ppindex)->nHeight); + const CBlockIndex& last_accepted{**ppindex}; + const int64_t blocks_left{(GetTime() - last_accepted.GetBlockTime()) / chainparams.GetConsensus().nPowTargetSpacing}; + const double progress{100.0 * last_accepted.nHeight / (last_accepted.nHeight + blocks_left)}; + LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", last_accepted.nHeight, progress); } } return true; @@ -3591,7 +3626,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block !ContextualCheckBlock(block, state, m_params.GetConsensus(), pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); + m_blockman.m_dirty_blockindex.insert(pindex); } return error("%s: %s", __func__, state.ToString()); } @@ -3604,7 +3639,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp); + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3992,10 +4027,6 @@ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) chainman.Unload(); pindexBestHeader = nullptr; if (mempool) mempool->clear(); - vinfoBlockFile.clear(); - nLastBlockFile = 0; - setDirtyBlockIndex.clear(); - setDirtyFileInfo.clear(); g_versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); @@ -4039,9 +4070,10 @@ bool CChainState::LoadGenesisBlock() try { const CBlock& block = m_params.GenesisBlock(); - FlatFilePos blockPos = SaveBlockToDisk(block, 0, m_chain, m_params, nullptr); - if (blockPos.IsNull()) + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, m_params, nullptr)}; + if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); + } CBlockIndex *pindex = m_blockman.AddToBlockIndex(block); ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { @@ -4879,7 +4911,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( // about the snapshot_chainstate. CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); - if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, breakpoint_fnc)) { + if (!GetUTXOStats(snapshot_coinsdb, m_blockman, stats, breakpoint_fnc)) { LogPrintf("[snapshot] failed to generate coins stats\n"); return false; } @@ -4929,7 +4961,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( index->nStatus |= BLOCK_OPT_WITNESS; } - setDirtyBlockIndex.insert(index); + m_blockman.m_dirty_blockindex.insert(index); // Changes to the block index will be flushed to disk after this call // returns in `ActivateSnapshot()`, when `MaybeRebalanceCaches()` is // called, since we've added a snapshot chainstate and therefore will diff --git a/src/validation.h b/src/validation.h index 16f4bfe741..6e9912cada 100644 --- a/src/validation.h +++ b/src/validation.h @@ -43,12 +43,14 @@ class CBlockTreeDB; class CChainParams; class CTxMemPool; class ChainstateManager; -class SnapshotMetadata; struct ChainTxData; struct DisconnectedBlockTransactions; struct PrecomputedTransactionData; struct LockPoints; struct AssumeutxoData; +namespace node { +class SnapshotMetadata; +} // namespace node /** Default for -minrelaytxfee, minimum relay fee for transactions */ static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1000; @@ -161,8 +163,12 @@ struct MempoolAcceptResult { VALID, //!> Fully validated, valid. INVALID, //!> Invalid. MEMPOOL_ENTRY, //!> Valid, transaction was already in the mempool. + DIFFERENT_WITNESS, //!> Not validated. A same-txid-different-witness tx (see m_other_wtxid) already exists in the mempool and was not replaced. }; + /** Result type. Present in all MempoolAcceptResults. */ const ResultType m_result_type; + + /** Contains information about why the transaction failed. */ const TxValidationState m_state; // The following fields are only present when m_result_type = ResultType::VALID or MEMPOOL_ENTRY @@ -173,6 +179,10 @@ struct MempoolAcceptResult { /** Raw base fees in satoshis. */ const std::optional<CAmount> m_base_fees; + // The following field is only present when m_result_type = ResultType::DIFFERENT_WITNESS + /** The wtxid of the transaction in the mempool which has the same txid but different witness. */ + const std::optional<uint256> m_other_wtxid; + static MempoolAcceptResult Failure(TxValidationState state) { return MempoolAcceptResult(state); } @@ -185,6 +195,10 @@ struct MempoolAcceptResult { return MempoolAcceptResult(vsize, fees); } + static MempoolAcceptResult MempoolTxDifferentWitness(const uint256& other_wtxid) { + return MempoolAcceptResult(other_wtxid); + } + // Private constructors. Use static methods MempoolAcceptResult::Success, etc. to construct. private: /** Constructor for failure case */ @@ -201,6 +215,10 @@ private: /** Constructor for already-in-mempool case. It wouldn't replace any transactions. */ explicit MempoolAcceptResult(int64_t vsize, CAmount fees) : m_result_type(ResultType::MEMPOOL_ENTRY), m_vsize{vsize}, m_base_fees(fees) {} + + /** Constructor for witness-swapped case. */ + explicit MempoolAcceptResult(const uint256& other_wtxid) + : m_result_type(ResultType::DIFFERENT_WITNESS), m_other_wtxid(other_wtxid) {} }; /** @@ -210,7 +228,7 @@ struct PackageMempoolAcceptResult { const PackageValidationState m_state; /** - * Map from (w)txid to finished MempoolAcceptResults. The client is responsible + * Map from wtxid to finished MempoolAcceptResults. The client is responsible * for keeping track of the transaction objects themselves. If a result is not * present, it means validation was unfinished for that transaction. If there * was a package-wide error (see result in m_state), m_tx_results will be empty. @@ -476,7 +494,7 @@ protected: public: //! Reference to a BlockManager instance which itself is shared across all //! CChainState instances. - BlockManager& m_blockman; + node::BlockManager& m_blockman; /** Chain parameters for this chainstate */ const CChainParams& m_params; @@ -488,7 +506,7 @@ public: explicit CChainState( CTxMemPool* mempool, - BlockManager& blockman, + node::BlockManager& blockman, ChainstateManager& chainman, std::optional<uint256> from_snapshot_blockhash = std::nullopt); @@ -535,7 +553,7 @@ public: * chainstates) and as good as our current tip or better. Entries may be failed, * though, and pruning nodes may be missing the data for the block. */ - std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; + std::set<CBlockIndex*, node::CBlockIndexWorkComparator> setBlockIndexCandidates; //! @returns A reference to the in-memory cache of the UTXO set. CCoinsViewCache& CoinsTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -803,13 +821,13 @@ private: bool m_snapshot_validated{false}; CBlockIndex* m_best_invalid; - friend bool BlockManager::LoadBlockIndex(const Consensus::Params&, ChainstateManager&); + friend bool node::BlockManager::LoadBlockIndex(const Consensus::Params&, ChainstateManager&); //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, CAutoFile& coins_file, - const SnapshotMetadata& metadata); + const node::SnapshotMetadata& metadata); /** * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure @@ -826,7 +844,7 @@ public: std::thread m_load_block; //! A single BlockManager instance is shared across each constructed //! chainstate to avoid duplicating block metadata. - BlockManager m_blockman GUARDED_BY(::cs_main); + node::BlockManager m_blockman; /** * In order to efficiently track invalidity of headers, we keep the set of @@ -886,7 +904,7 @@ public: //! - Move the new chainstate to `m_snapshot_chainstate` and make it our //! ChainstateActive(). [[nodiscard]] bool ActivateSnapshot( - CAutoFile& coins_file, const SnapshotMetadata& metadata, bool in_memory); + CAutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); //! The most-work chain. CChainState& ActiveChainstate() const; @@ -894,7 +912,7 @@ public: int ActiveHeight() const { return ActiveChain().Height(); } CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } - BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + node::BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return m_blockman.m_block_index; } diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 5b264ac4ad..e0be914a2b 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -15,6 +15,7 @@ #include <sys/stat.h> #endif +namespace wallet { namespace { //! Make sure database has a unique fileid within the environment. If it @@ -846,3 +847,4 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con status = DatabaseStatus::SUCCESS; return db; } +} // namespace wallet diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 0cf9843cc8..b924890d81 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -31,6 +31,7 @@ struct bilingual_str; +namespace wallet { static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; static const bool DEFAULT_WALLET_PRIVDB = true; @@ -229,5 +230,6 @@ bool BerkeleyDatabaseSanityCheck(); //! Return object giving access to Berkeley database at specified path. std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); +} // namespace wallet #endif // BITCOIN_WALLET_BDB_H diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index bf582e1cc1..3b3c1f8da4 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -6,7 +6,9 @@ #include <util/system.h> +namespace wallet { CCoinControl::CCoinControl() { m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS); } +} // namespace wallet diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index cb30b09b87..5ef2295c88 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -18,6 +18,7 @@ #include <map> #include <set> +namespace wallet { const int DEFAULT_MIN_DEPTH = 0; const int DEFAULT_MAX_DEPTH = 9999999; @@ -118,5 +119,6 @@ private: std::set<COutPoint> setSelected; std::map<COutPoint, CTxOut> m_external_txouts; }; +} // namespace wallet #endif // BITCOIN_WALLET_COINCONTROL_H diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 3b48a8957e..23faad027f 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -13,6 +13,7 @@ #include <numeric> #include <optional> +namespace wallet { // Descending order comparator struct { bool operator()(const OutputGroup& a, const OutputGroup& b) const @@ -429,3 +430,4 @@ bool SelectionResult::operator<(SelectionResult other) const // As this operator is only used in std::min_element, we want the result that has more inputs when waste are equal. return *m_waste < *other.m_waste || (*m_waste == *other.m_waste && m_selected_inputs.size() > other.m_selected_inputs.size()); } +} // namespace wallet diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index a0001351f7..496a026999 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -12,6 +12,7 @@ #include <optional> +namespace wallet { //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -249,5 +250,6 @@ std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& ut // Original coin selection algorithm as a fallback std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue); +} // namespace wallet #endif // BITCOIN_WALLET_COINSELECTION_H diff --git a/src/wallet/context.cpp b/src/wallet/context.cpp index 09b2f30467..800aa5bf9c 100644 --- a/src/wallet/context.cpp +++ b/src/wallet/context.cpp @@ -4,5 +4,7 @@ #include <wallet/context.h> +namespace wallet { WalletContext::WalletContext() {} WalletContext::~WalletContext() {} +} // namespace wallet diff --git a/src/wallet/context.h b/src/wallet/context.h index 0c11182ee6..57a6ed77f7 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -13,12 +13,13 @@ #include <vector> class ArgsManager; -class CWallet; namespace interfaces { class Chain; class Wallet; } // namespace interfaces +namespace wallet { +class CWallet; using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; //! WalletContext struct containing references to state shared between CWallet @@ -46,5 +47,6 @@ struct WalletContext { WalletContext(); ~WalletContext(); }; +} // namespace wallet #endif // BITCOIN_WALLET_CONTEXT_H diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 683b55a408..cd414b3d44 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -10,6 +10,7 @@ #include <vector> +namespace wallet { int CCrypter::BytesToKeySHA512AES(const std::vector<unsigned char>& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const { // This mimics the behavior of openssl's EVP_BytesToKey with an aes256cbc @@ -136,3 +137,4 @@ bool DecryptKey(const CKeyingMaterial& vMasterKey, const std::vector<unsigned ch key.Set(vchSecret.begin(), vchSecret.end(), vchPubKey.IsCompressed()); return key.VerifyPubKey(vchPubKey); } +} // namespace wallet diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index f7325541a9..4d325c7557 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -10,6 +10,7 @@ #include <script/signingprovider.h> +namespace wallet { const unsigned int WALLET_CRYPTO_KEY_SIZE = 32; const unsigned int WALLET_CRYPTO_SALT_SIZE = 8; const unsigned int WALLET_CRYPTO_IV_SIZE = 16; @@ -105,5 +106,6 @@ public: bool EncryptSecret(const CKeyingMaterial& vMasterKey, const CKeyingMaterial &vchPlaintext, const uint256& nIV, std::vector<unsigned char> &vchCiphertext); bool DecryptSecret(const CKeyingMaterial& vMasterKey, const std::vector<unsigned char>& vchCiphertext, const uint256& nIV, CKeyingMaterial& vchPlaintext); bool DecryptKey(const CKeyingMaterial& vMasterKey, const std::vector<unsigned char>& vchCryptedSecret, const CPubKey& vchPubKey, CKey& key); +} // namespace wallet #endif // BITCOIN_WALLET_CRYPTER_H diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 6437e1d7f0..414d0ef5c3 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -10,6 +10,7 @@ #include <string> +namespace wallet { std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) { std::vector<fs::path> paths; @@ -132,3 +133,4 @@ bool IsSQLiteFile(const fs::path& path) // Check the application id matches our network magic return memcmp(Params().MessageStart(), app_id, 4) == 0; } +} // namespace wallet diff --git a/src/wallet/db.h b/src/wallet/db.h index 7acc27e2b4..5825b00e3a 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -18,6 +18,7 @@ struct bilingual_str; +namespace wallet { void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename); /** RAII class that provides access to a WalletDatabase */ @@ -232,5 +233,6 @@ fs::path BDBDataFile(const fs::path& path); fs::path SQLiteDataFile(const fs::path& path); bool IsBDBFile(const fs::path& path); bool IsSQLiteFile(const fs::path& path); +} // namespace wallet #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 645576ef91..7dfb1d8839 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -7,6 +7,7 @@ #include <util/translation.h> #include <wallet/wallet.h> +namespace wallet { static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP"; uint32_t DUMP_VERSION = 1; @@ -214,6 +215,11 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling if (key == "checksum") { std::vector<unsigned char> parsed_checksum = ParseHex(value); + if (parsed_checksum.size() != checksum.size()) { + error = Untranslated("Error: Checksum is not the correct size"); + ret = false; + break; + } std::copy(parsed_checksum.begin(), parsed_checksum.end(), checksum.begin()); break; } @@ -279,3 +285,4 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling return ret; } +} // namespace wallet diff --git a/src/wallet/dump.h b/src/wallet/dump.h index d0a4f5ef1d..4effab3bed 100644 --- a/src/wallet/dump.h +++ b/src/wallet/dump.h @@ -7,11 +7,12 @@ #include <fs.h> -class CWallet; - struct bilingual_str; +namespace wallet { +class CWallet; bool DumpWallet(CWallet& wallet, bilingual_str& error); bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); +} // namespace wallet #endif // BITCOIN_WALLET_DUMP_H diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index ad89478874..9d5f58b784 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -13,6 +13,7 @@ #include <utility> #include <vector> +namespace wallet { bool ExternalSignerScriptPubKeyMan::SetupDescriptor(std::unique_ptr<Descriptor> desc) { LOCK(cs_desc_man); @@ -82,3 +83,4 @@ TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransact if (finalize) FinalizePSBT(psbt); // This won't work in a multisig setup return TransactionError::OK; } +} // namespace wallet diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index 92c1ccb401..9918979a81 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -9,6 +9,7 @@ #include <memory> +namespace wallet { class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan { public: @@ -30,4 +31,5 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; }; +} // namespace wallet #endif // BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 242a3c2847..3552c14160 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -16,6 +16,7 @@ #include <wallet/spend.h> #include <wallet/wallet.h> +namespace wallet { //! Check whether transaction has descendant in wallet or mempool, or has been //! mined, or conflicts with a mined transaction. Return a feebumper::Result. static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<bilingual_str>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) @@ -282,3 +283,4 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti } } // namespace feebumper +} // namespace wallet diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 50577c9d3e..191878a137 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -7,13 +7,15 @@ #include <primitives/transaction.h> -class CWallet; -class CWalletTx; class uint256; -class CCoinControl; enum class FeeEstimateMode; struct bilingual_str; +namespace wallet { +class CCoinControl; +class CWallet; +class CWalletTx; + namespace feebumper { enum class Result @@ -54,5 +56,6 @@ Result CommitTransaction(CWallet& wallet, uint256& bumped_txid); } // namespace feebumper +} // namespace wallet #endif // BITCOIN_WALLET_FEEBUMPER_H diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 429101e774..6f81fa30a1 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -9,6 +9,7 @@ #include <wallet/wallet.h> +namespace wallet { CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes) { return GetRequiredFeeRate(wallet).GetFee(nTxBytes); @@ -90,3 +91,4 @@ CFeeRate GetDiscardRate(const CWallet& wallet) discard_rate = std::max(discard_rate, wallet.chain().relayDustFee()); return discard_rate; } +} // namespace wallet diff --git a/src/wallet/fees.h b/src/wallet/fees.h index ad35670d4e..af7f759553 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -8,11 +8,13 @@ #include <consensus/amount.h> -class CCoinControl; class CFeeRate; -class CWallet; struct FeeCalculation; +namespace wallet { +class CCoinControl; +class CWallet; + /** * Return the minimum required absolute fee for this size * based on the required fee rate @@ -41,5 +43,6 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr * Return the maximum feerate for discarding change. */ CFeeRate GetDiscardRate(const CWallet& wallet); +} // namespace wallet #endif // BITCOIN_WALLET_FEES_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 2f73ec3d4c..7a83dbc35d 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -23,6 +23,9 @@ #include <wallet/wallet.h> #include <walletinitinterface.h> +using node::NodeContext; + +namespace wallet { class WalletInit : public WalletInitInterface { public: @@ -39,8 +42,6 @@ public: void Construct(NodeContext& node) const override; }; -const WalletInitInterface& g_wallet_init_interface = WalletInit(); - void WalletInit::AddWalletOptions(ArgsManager& argsman) const { argsman.AddArg("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", \"bech32\", or \"bech32m\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); @@ -137,3 +138,6 @@ void WalletInit::Construct(NodeContext& node) const node.wallet_loader = wallet_loader.get(); node.chain_clients.emplace_back(std::move(wallet_loader)); } +} // namespace wallet + +const WalletInitInterface& g_wallet_init_interface = wallet::WalletInit(); diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 64775a5ddb..b1466869b9 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -557,7 +557,7 @@ public: options.require_existing = true; return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } - std::unique_ptr<Wallet> restoreWallet(const std::string& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) override + std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) override { DatabaseStatus status; @@ -598,7 +598,7 @@ public: } // namespace wallet namespace interfaces { -std::unique_ptr<Wallet> MakeWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet) { return wallet ? std::make_unique<wallet::WalletImpl>(context, wallet) : nullptr; } +std::unique_ptr<Wallet> MakeWallet(wallet::WalletContext& context, const std::shared_ptr<wallet::CWallet>& wallet) { return wallet ? std::make_unique<wallet::WalletImpl>(context, wallet) : nullptr; } std::unique_ptr<WalletLoader> MakeWalletLoader(Chain& chain, ArgsManager& args) { diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 8527b460df..4ef4ef98ac 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -12,9 +12,11 @@ #include <cstdint> #include <type_traits> -class CWallet; class CScript; +namespace wallet { +class CWallet; + /** * IsMine() return codes, which depend on ScriptPubKeyMan implementation. * Not every ScriptPubKeyMan covers all types, please refer to @@ -66,5 +68,6 @@ struct CachableAmount m_value[filter] = value; } }; +} // namespace wallet #endif // BITCOIN_WALLET_ISMINE_H diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 4d59e17709..e6f96074d5 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -19,6 +19,7 @@ #include <univalue.h> +namespace wallet { bool VerifyWallets(WalletContext& context) { interfaces::Chain& chain = *context.chain; @@ -28,7 +29,7 @@ bool VerifyWallets(WalletContext& context) fs::path wallet_dir = fs::PathFromString(args.GetArg("-walletdir", "")); boost::system::error_code error; // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory - fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); + fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error).remove_trailing_separator(); if (error || !fs::exists(wallet_dir)) { chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), fs::PathToString(wallet_dir))); return false; @@ -169,3 +170,4 @@ void UnloadWallets(WalletContext& context) UnloadWallet(std::move(wallet)); } } +} // namespace wallet diff --git a/src/wallet/load.h b/src/wallet/load.h index e207bc2e09..5c2bbdabe4 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -11,12 +11,14 @@ class ArgsManager; class CScheduler; -struct WalletContext; namespace interfaces { class Chain; } // namespace interfaces +namespace wallet { +struct WalletContext; + //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. bool VerifyWallets(WalletContext& context); @@ -34,5 +36,6 @@ void StopWallets(WalletContext& context); //! Close all wallets. void UnloadWallets(WalletContext& context); +} // namespace wallet #endif // BITCOIN_WALLET_LOAD_H diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 2fb274b55f..e598d6f979 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -8,6 +8,7 @@ #include <wallet/transaction.h> #include <wallet/wallet.h> +namespace wallet { isminetype InputIsMine(const CWallet& wallet, const CTxIn &txin) { AssertLockHeld(wallet.cs_wallet); @@ -473,3 +474,4 @@ std::set< std::set<CTxDestination> > GetAddressGroupings(const CWallet& wallet) return ret; } +} // namespace wallet diff --git a/src/wallet/receive.h b/src/wallet/receive.h index f659955fc6..d7705b5262 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -10,6 +10,7 @@ #include <wallet/transaction.h> #include <wallet/wallet.h> +namespace wallet { isminetype InputIsMine(const CWallet& wallet, const CTxIn& txin) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); /** Returns whether all of the inputs match the filter */ @@ -60,5 +61,6 @@ Balance GetBalance(const CWallet& wallet, int min_depth = 0, bool avoid_reuse = std::map<CTxDestination, CAmount> GetAddressBalances(const CWallet& wallet); std::set<std::set<CTxDestination>> GetAddressGroupings(const CWallet& wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +} // namespace wallet #endif // BITCOIN_WALLET_RECEIVE_H diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 2db47bc855..51587a64a3 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -13,6 +13,7 @@ #include <univalue.h> +namespace wallet { RPCHelpMan getnewaddress() { return RPCHelpMan{"getnewaddress", @@ -802,3 +803,4 @@ RPCHelpMan walletdisplayaddress() }; } #endif // ENABLE_EXTERNAL_SIGNER +} // namespace wallet diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 0029d1b09c..c0912ffc70 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -31,6 +31,7 @@ using interfaces::FoundBlock; +namespace wallet { std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; for (const unsigned char c : str) { @@ -1887,7 +1888,7 @@ RPCHelpMan restorewallet() bilingual_str error; std::vector<bilingual_str> warnings; - const std::shared_ptr<CWallet> wallet = RestoreWallet(context, fs::PathToString(backup_file), wallet_name, load_on_start, status, error, warnings); + const std::shared_ptr<CWallet> wallet = RestoreWallet(context, backup_file, wallet_name, load_on_start, status, error, warnings); HandleWalletError(wallet, status, error); @@ -1900,3 +1901,4 @@ RPCHelpMan restorewallet() }, }; } +} // namespace wallet diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index f3294b4570..f10de11662 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -15,6 +15,7 @@ #include <univalue.h> +namespace wallet { static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { std::set<CTxDestination> address_set; @@ -731,3 +732,4 @@ RPCHelpMan listunspent() }, }; } +} // namespace wallet diff --git a/src/wallet/rpc/encrypt.cpp b/src/wallet/rpc/encrypt.cpp index e659f434a3..2b6a2a198d 100644 --- a/src/wallet/rpc/encrypt.cpp +++ b/src/wallet/rpc/encrypt.cpp @@ -7,6 +7,7 @@ #include <wallet/wallet.h> +namespace wallet { RPCHelpMan walletpassphrase() { return RPCHelpMan{"walletpassphrase", @@ -246,3 +247,4 @@ RPCHelpMan encryptwallet() }, }; } +} // namespace wallet diff --git a/src/wallet/rpc/signmessage.cpp b/src/wallet/rpc/signmessage.cpp index bb8d7fc13f..438d290030 100644 --- a/src/wallet/rpc/signmessage.cpp +++ b/src/wallet/rpc/signmessage.cpp @@ -10,6 +10,7 @@ #include <univalue.h> +namespace wallet { RPCHelpMan signmessage() { return RPCHelpMan{"signmessage", @@ -66,3 +67,4 @@ RPCHelpMan signmessage() }, }; } +} // namespace wallet diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 978174b340..cae3542a5e 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -19,6 +19,7 @@ #include <univalue.h> +namespace wallet { static void ParseRecipients(const UniValue& address_amounts, const UniValue& subtract_fee_outputs, std::vector<CRecipient> &recipients) { std::set<CTxDestination> destinations; int i = 0; @@ -1367,3 +1368,4 @@ RPCHelpMan walletcreatefundedpsbt() }, }; } +} // namespace wallet diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 8a1c0885ac..d9034808f4 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -13,6 +13,7 @@ using interfaces::FoundBlock; +namespace wallet { static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue& entry) { interfaces::Chain& chain = wallet.chain(); @@ -958,3 +959,4 @@ RPCHelpMan abortrescan() }, }; } +} // namespace wallet diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 9a40a67ee5..59683c5fd8 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -12,6 +12,7 @@ #include <univalue.h> +namespace wallet { static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; @@ -147,4 +148,5 @@ void HandleWalletError(const std::shared_ptr<CWallet> wallet, DatabaseStatus& st } throw JSONRPCError(code, error.original); } -}
\ No newline at end of file +} +} // namespace wallet diff --git a/src/wallet/rpc/util.h b/src/wallet/rpc/util.h index 5b00d2abcb..7b810eb06e 100644 --- a/src/wallet/rpc/util.h +++ b/src/wallet/rpc/util.h @@ -10,12 +10,14 @@ #include <string> #include <vector> +class JSONRPCRequest; +class UniValue; struct bilingual_str; + +namespace wallet { class CWallet; -enum class DatabaseStatus; -class JSONRPCRequest; class LegacyScriptPubKeyMan; -class UniValue; +enum class DatabaseStatus; struct WalletContext; extern const std::string HELP_REQUIRING_PASSPHRASE; @@ -39,5 +41,6 @@ bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWallet& wal std::string LabelFromValue(const UniValue& value); void HandleWalletError(const std::shared_ptr<CWallet> wallet, DatabaseStatus& status, bilingual_str& error); +} // namespace wallet #endif // BITCOIN_WALLET_RPC_UTIL_H diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index bbe40feec0..33ec715b51 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -18,6 +18,7 @@ #include <univalue.h> +namespace wallet { /** Checks if a CKey is in the given CWallet compressed or otherwise*/ bool HaveKey(const SigningProvider& wallet, const CKey& key) { @@ -729,3 +730,4 @@ static const CRPCCommand commands[] = // clang-format on return commands; } +} // namespace wallet diff --git a/src/wallet/rpc/wallet.h b/src/wallet/rpc/wallet.h index 537d9b2358..423fc892b2 100644 --- a/src/wallet/rpc/wallet.h +++ b/src/wallet/rpc/wallet.h @@ -9,6 +9,8 @@ class CRPCCommand; +namespace wallet { Span<const CRPCCommand> GetWalletRPCCommands(); +} // namespace wallet #endif // BITCOIN_WALLET_RPC_WALLET_H diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 3859d67b6d..1ecc96fe0e 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -11,6 +11,7 @@ #include <wallet/wallet.h> #include <wallet/walletdb.h> +namespace wallet { /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ @@ -165,3 +166,4 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v return fSuccess; } +} // namespace wallet diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h index 5a8538f942..332aceb262 100644 --- a/src/wallet/salvage.h +++ b/src/wallet/salvage.h @@ -11,6 +11,8 @@ struct bilingual_str; +namespace wallet { bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings); +} // namespace wallet #endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 04a52bc86f..7218ed11dc 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -17,6 +17,7 @@ #include <optional> +namespace wallet { //! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; @@ -2359,3 +2360,4 @@ bool DescriptorScriptPubKeyMan::CanUpdateToWalletDescriptor(const WalletDescript return true; } +} // namespace wallet diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 8f511b68b5..6eda133771 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -25,6 +25,7 @@ enum class OutputType; struct bilingual_str; +namespace wallet { // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. // It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as // wallet flags, wallet version, encryption keys, encryption status, and the database itself. This allows a @@ -631,5 +632,6 @@ public: void UpgradeDescriptorCache(); }; +} // namespace wallet #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5d9cc7bf6b..d87bdc8679 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -21,6 +21,7 @@ using interfaces::FoundBlock; +namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig) @@ -665,8 +666,6 @@ static bool CreateTransactionInternal( } // Create change script that will be used if we need change - // TODO: pass in scriptChange instead of reservedest so - // change transaction isn't always pay-to-bitcoin-address CScript scriptChange; // coin control: send change to custom address @@ -1030,3 +1029,4 @@ bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, return true; } +} // namespace wallet diff --git a/src/wallet/spend.h b/src/wallet/spend.h index 268bcfc033..4453fb2762 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -10,6 +10,7 @@ #include <wallet/transaction.h> #include <wallet/wallet.h> +namespace wallet { /** Get the marginal bytes if spending the specified output from this transaction */ int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); @@ -142,5 +143,6 @@ bool CreateTransaction(CWallet& wallet, const std::vector<CRecipient>& vecSend, * calling CreateTransaction(); */ bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); +} // namespace wallet #endif // BITCOIN_WALLET_SPEND_H diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 3bea0de99f..07e387f177 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -20,6 +20,7 @@ #include <utility> #include <vector> +namespace wallet { static constexpr int32_t WALLET_SCHEMA_VERSION = 0; static Mutex g_sqlite_mutex; @@ -578,3 +579,4 @@ std::string SQLiteDatabaseVersion() { return std::string(sqlite3_libversion()); } +} // namespace wallet diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 70ab4f797a..3ed598d0d2 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -10,6 +10,8 @@ #include <sqlite3.h> struct bilingual_str; + +namespace wallet { class SQLiteDatabase; /** RAII class that provides access to a WalletDatabase */ @@ -116,5 +118,6 @@ public: std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); std::string SQLiteDatabaseVersion(); +} // namespace wallet #endif // BITCOIN_WALLET_SQLITE_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 9ab3c81078..b9f12158ca 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -18,6 +18,7 @@ #include <boost/test/unit_test.hpp> #include <random> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) // how many times to run all the tests to have a chance to catch errors that only show up with particular random shuffles @@ -807,3 +808,4 @@ BOOST_AUTO_TEST_CASE(waste_test) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index d9359bc171..825382fe59 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -11,6 +11,7 @@ #include <wallet/bdb.h> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, std::string& database_filename) @@ -77,3 +78,4 @@ BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index 0601c492cd..1c16da25bd 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -18,6 +18,7 @@ #include <string> #include <vector> +namespace wallet { namespace { const TestingSetup* g_setup; @@ -168,3 +169,4 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) } } } // namespace +} // namespace wallet diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 439489ab59..b455ab9d9e 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -9,6 +9,7 @@ #include <wallet/test/init_test_fixture.h> +namespace wallet { InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { m_wallet_loader = MakeWalletLoader(*m_node.chain, *Assert(m_node.args)); @@ -48,3 +49,4 @@ void InitWalletDirTestingSetup::SetWalletDir(const fs::path& walletdir_path) { gArgs.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); } +} // namespace wallet diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index ccad629543..df5819fd1d 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -11,6 +11,7 @@ #include <test/util/setup_common.h> +namespace wallet { struct InitWalletDirTestingSetup: public BasicTestingSetup { explicit InitWalletDirTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~InitWalletDirTestingSetup(); @@ -23,3 +24,4 @@ struct InitWalletDirTestingSetup: public BasicTestingSetup { }; #endif // BITCOIN_WALLET_TEST_INIT_TEST_FIXTURE_H +} // namespace wallet diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 439b17fe13..c1cae5c5f6 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -10,6 +10,7 @@ #include <util/system.h> #include <wallet/test/init_test_fixture.h> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(init_tests, InitWalletDirTestingSetup) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) @@ -83,3 +84,4 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index a9b3f1cbfd..dd5cd0af46 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -13,6 +13,7 @@ #include <boost/test/unit_test.hpp> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(ismine_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(ismine_standard) @@ -417,3 +418,4 @@ BOOST_AUTO_TEST_CASE(ismine_standard) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 4c3c23599e..b953f402a2 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -11,6 +11,7 @@ #include <test/util/setup_common.h> #include <wallet/test/wallet_test_fixture.h> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) static void import_descriptor(CWallet& wallet, const std::string& descriptor) @@ -145,3 +146,4 @@ BOOST_AUTO_TEST_CASE(parse_hd_keypath) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index f51648ec3b..a524b85ccb 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -10,6 +10,7 @@ #include <boost/test/unit_test.hpp> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(scriptpubkeyman_tests, BasicTestingSetup) // Test LegacyScriptPubKeyMan::CanProvide behavior, making sure it returns true @@ -39,3 +40,4 @@ BOOST_AUTO_TEST_CASE(CanProvide) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index 926f28686d..b2a0697c21 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -12,6 +12,7 @@ #include <boost/test/unit_test.hpp> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(spend_tests, WalletTestingSetup) BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) @@ -63,3 +64,4 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 93a3404d2c..aa3121511d 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -15,6 +15,7 @@ #include <memory> +namespace wallet { std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key) { auto wallet = std::make_unique<CWallet>(&chain, "", args, CreateMockWalletDatabase()); @@ -44,3 +45,4 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc BOOST_CHECK(result.last_failed_block.IsNull()); return wallet; } +} // namespace wallet diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index 3adb82b85f..712d0251cd 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -10,11 +10,14 @@ class ArgsManager; class CChain; class CKey; -class CWallet; namespace interfaces { class Chain; } // namespace interfaces +namespace wallet { +class CWallet; + std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key); +} // namespace wallet #endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 5b421840e0..166e27bab9 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -10,6 +10,7 @@ #include <boost/test/unit_test.hpp> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(wallet_crypto_tests, BasicTestingSetup) class TestCrypter @@ -124,3 +125,4 @@ BOOST_AUTO_TEST_CASE(decrypt) { } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index ad24662fb6..cb006dea3a 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -6,6 +6,7 @@ #include <scheduler.h> +namespace wallet { WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), m_wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()) @@ -19,3 +20,4 @@ WalletTestingSetup::~WalletTestingSetup() { if (m_node.scheduler) m_node.scheduler->stop(); } +} // namespace wallet diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index cb6a8402dd..d4b855b145 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -15,6 +15,7 @@ #include <memory> +namespace wallet { /** Testing setup and teardown for wallet. */ struct WalletTestingSetup : public TestingSetup { @@ -25,5 +26,6 @@ struct WalletTestingSetup : public TestingSetup { CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; +} // namespace wallet #endif // BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 48998594f1..bb6021b857 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -30,6 +30,10 @@ #include <boost/test/unit_test.hpp> #include <univalue.h> +using node::MAX_BLOCKFILE_SIZE; +using node::UnlinkPrunedFiles; + +namespace wallet { RPCHelpMan importmulti(); RPCHelpMan dumpwallet(); RPCHelpMan importwallet(); @@ -92,7 +96,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) { // Cap last block file size, and mine new block in a new block file. CBlockIndex* oldTip = m_node.chainman->ActiveChain().Tip(); - GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE; + WITH_LOCK(::cs_main, m_node.chainman->m_blockman.GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE); CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = m_node.chainman->ActiveChain().Tip(); @@ -193,7 +197,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) { // Cap last block file size, and mine new block in a new block file. CBlockIndex* oldTip = m_node.chainman->ActiveChain().Tip(); - GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE; + WITH_LOCK(::cs_main, m_node.chainman->m_blockman.GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE); CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = m_node.chainman->ActiveChain().Tip(); @@ -289,7 +293,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) request.params.setArray(); request.params.push_back(backup_file); - ::dumpwallet().HandleRequest(request); + wallet::dumpwallet().HandleRequest(request); RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); } @@ -308,7 +312,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) request.params.push_back(backup_file); AddWallet(context, wallet); wallet->SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); - ::importwallet().HandleRequest(request); + wallet::importwallet().HandleRequest(request); RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); BOOST_CHECK_EQUAL(wallet->mapWallet.size(), 3U); @@ -851,3 +855,4 @@ BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/wallet_transaction_tests.cpp b/src/wallet/test/wallet_transaction_tests.cpp index 5ef2904f66..9f56248614 100644 --- a/src/wallet/test/wallet_transaction_tests.cpp +++ b/src/wallet/test/wallet_transaction_tests.cpp @@ -8,6 +8,7 @@ #include <boost/test/unit_test.hpp> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(wallet_transaction_tests, WalletTestingSetup) BOOST_AUTO_TEST_CASE(roundtrip) @@ -22,3 +23,4 @@ BOOST_AUTO_TEST_CASE(roundtrip) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/walletdb_tests.cpp b/src/wallet/test/walletdb_tests.cpp index 558121ae42..e251a3a0e4 100644 --- a/src/wallet/test/walletdb_tests.cpp +++ b/src/wallet/test/walletdb_tests.cpp @@ -9,6 +9,7 @@ #include <boost/test/unit_test.hpp> +namespace wallet { BOOST_FIXTURE_TEST_SUITE(walletdb_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(walletdb_readkeyvalue) @@ -27,3 +28,4 @@ BOOST_AUTO_TEST_CASE(walletdb_readkeyvalue) } BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index a926c0ecc1..a46846c1d4 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -4,6 +4,7 @@ #include <wallet/transaction.h> +namespace wallet { bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { CMutableTransaction tx1 {*this->tx}; @@ -23,3 +24,4 @@ int64_t CWalletTx::GetTxTime() const int64_t n = nTimeSmart; return n ? n : nTimeReceived; } +} // namespace wallet diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 52d72cccf3..00f9c9f154 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -19,6 +19,7 @@ #include <variant> #include <vector> +namespace wallet { //! State of transaction confirmed in a block. struct TxStateConfirmed { uint256 confirmed_block_hash; @@ -303,5 +304,6 @@ public: CWalletTx(CWalletTx const &) = delete; void operator=(CWalletTx const &x) = delete; }; +} // namespace wallet #endif // BITCOIN_WALLET_TRANSACTION_H diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 267710e8c7..3fcb086d2d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -48,6 +48,7 @@ using interfaces::FoundBlock; +namespace wallet { const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ {WALLET_FLAG_AVOID_REUSE, "You need to rescan the blockchain in order to correctly mark used " @@ -357,12 +358,12 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& return wallet; } -std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const std::string& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; options.require_existing = true; - if (!fs::exists(fs::u8path(backup_file))) { + if (!fs::exists(backup_file)) { error = Untranslated("Backup file does not exist"); status = DatabaseStatus::FAILED_INVALID_BACKUP_FILE; return nullptr; @@ -3433,3 +3434,4 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat return spk_man; } +} // namespace wallet diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cbcdcaf3b8..00a1865a0e 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_WALLET_H #include <consensus/amount.h> +#include <fs.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <outputtype.h> @@ -40,12 +41,17 @@ #include <boost/signals2/signal.hpp> -struct WalletContext; using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; +class CScript; +enum class FeeEstimateMode; +struct FeeCalculation; struct bilingual_str; +namespace wallet { +struct WalletContext; + //! Explicitly unload and delete the wallet. //! Blocks the current thread after signaling the unload intent so that all //! wallet pointer owners release the wallet. @@ -60,7 +66,7 @@ std::vector<std::shared_ptr<CWallet>> GetWallets(WalletContext& context); std::shared_ptr<CWallet> GetWallet(WalletContext& context, const std::string& name); std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); -std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const std::string& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); +std::shared_ptr<CWallet> RestoreWallet(WalletContext& context, const fs::path& backup_file, const std::string& wallet_name, std::optional<bool> load_on_start, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, LoadWalletFn load_wallet); std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); @@ -107,10 +113,7 @@ static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; class CCoinControl; class COutput; -class CScript; class CWalletTx; -struct FeeCalculation; -enum class FeeEstimateMode; class ReserveDestination; //! Default for -addresstype @@ -936,5 +939,6 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); +} // namespace wallet #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 4b51d60597..9cef76d803 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -26,6 +26,7 @@ #include <optional> #include <string> +namespace wallet { namespace DBKeys { const std::string ACENTRY{"acentry"}; const std::string ACTIVEEXTERNALSPK{"activeexternalspk"}; @@ -1194,3 +1195,4 @@ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); #endif } +} // namespace wallet diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index be77fb03a2..7d38832aa5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -15,6 +15,18 @@ #include <string> #include <vector> +class CScript; +class uint160; +class uint256; +struct CBlockLocator; + +namespace wallet { +class CKeyPool; +class CMasterKey; +class CWallet; +class CWalletTx; +struct WalletContext; + /** * Overview of wallet database classes: * @@ -29,16 +41,6 @@ static const bool DEFAULT_FLUSHWALLET = true; -struct CBlockLocator; -struct WalletContext; -class CKeyPool; -class CMasterKey; -class CScript; -class CWallet; -class CWalletTx; -class uint160; -class uint256; - /** Error statuses for the wallet database */ enum class DBErrors { @@ -297,5 +299,6 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase(); /** Return object for accessing temporary in-memory database. */ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(); +} // namespace wallet #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 900651939c..9cd18dd0a5 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -16,6 +16,7 @@ #include <wallet/wallet.h> #include <wallet/walletutil.h> +namespace wallet { namespace WalletTool { // The standard wallet deleter function blocks on the validation interface @@ -219,3 +220,4 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) return true; } } // namespace WalletTool +} // namespace wallet diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index cdd728db33..9e0fe2b0ec 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -9,10 +9,12 @@ class ArgsManager; +namespace wallet { namespace WalletTool { bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command); } // namespace WalletTool +} // namespace wallet #endif // BITCOIN_WALLET_WALLETTOOL_H diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index d32fdb1fa8..ce276451c3 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -7,6 +7,7 @@ #include <logging.h> #include <util/system.h> +namespace wallet { fs::path GetWalletDir() { fs::path path; @@ -42,3 +43,4 @@ WalletFeature GetClosestWalletFeature(int version) } return static_cast<WalletFeature>(0); } +} // namespace wallet diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 06569d2f5f..788d41ceb7 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -10,6 +10,7 @@ #include <vector> +namespace wallet { /** (client) version numbers for particular wallet features */ enum WalletFeature { @@ -103,5 +104,6 @@ public: WalletDescriptor() {} WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} }; +} // namespace wallet #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h index 660b0eed5d..7624c2b16d 100644 --- a/src/walletinitinterface.h +++ b/src/walletinitinterface.h @@ -7,7 +7,9 @@ class ArgsManager; +namespace node { struct NodeContext; +} // namespace node class WalletInitInterface { public: @@ -18,7 +20,7 @@ public: /** Check wallet parameter interaction */ virtual bool ParameterInteraction() const = 0; /** Add wallets that should be opened to list of chain clients. */ - virtual void Construct(NodeContext& node) const = 0; + virtual void Construct(node::NodeContext& node) const = 0; virtual ~WalletInitInterface() {} }; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 2748741e2c..543db10612 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -23,6 +23,8 @@ #include <string> #include <utility> +using node::ReadBlockFromDisk; + static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; static const char *MSG_HASHBLOCK = "hashblock"; diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 33d6282961..3747b2a98d 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -28,6 +28,7 @@ from test_framework.messages import ( CTxIn, CTxOut, MAX_MONEY, + SEQUENCE_FINAL, ) from test_framework.blocktools import create_tx_with_script, MAX_BLOCK_SIGOPS from test_framework.script import ( @@ -77,7 +78,7 @@ class BadTxTemplate: def __init__(self, *, spend_tx=None, spend_block=None): self.spend_tx = spend_block.vtx[0] if spend_block else spend_tx self.spend_avail = sum(o.nValue for o in self.spend_tx.vout) - self.valid_txin = CTxIn(COutPoint(self.spend_tx.sha256, 0), b"", 0xffffffff) + self.valid_txin = CTxIn(COutPoint(self.spend_tx.sha256, 0), b"", SEQUENCE_FINAL) @abc.abstractmethod def get_tx(self, *args, **kwargs): @@ -137,7 +138,7 @@ class BadInputOutpointIndex(BadTxTemplate): bad_idx = num_indices + 100 tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256, bad_idx), b"", 0xffffffff)) + tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256, bad_idx), b"", SEQUENCE_FINAL)) tx.vout.append(CTxOut(0, basic_p2sh)) tx.calc_sha256() return tx @@ -175,7 +176,7 @@ class NonexistentInput(BadTxTemplate): def get_tx(self): tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256 + 1, 0), b"", 0xffffffff)) + tx.vin.append(CTxIn(COutPoint(self.spend_tx.sha256 + 1, 0), b"", SEQUENCE_FINAL)) tx.vin.append(self.valid_txin) tx.vout.append(CTxOut(1, basic_p2sh)) tx.calc_sha256() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index a3253763bd..462deeae32 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -23,6 +23,7 @@ from test_framework.messages import ( CTxIn, CTxOut, MAX_BLOCK_WEIGHT, + SEQUENCE_FINAL, uint256_from_compact, uint256_from_str, ) @@ -50,9 +51,13 @@ from test_framework.script_util import ( script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import ( + assert_equal, + assert_greater_than, +) from data import invalid_txs + # Use this class for tests that require behavior other than normal p2p behavior. # For now, it is used to serialize a bloated varint (b64). class CBrokenBlock(CBlock): @@ -801,7 +806,7 @@ class FullBlockTest(BitcoinTestFramework): b58 = self.next_block(58, spend=out[17]) tx = CTransaction() assert len(out[17].vout) < 42 - tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), 0xffffffff)) + tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), SEQUENCE_FINAL)) tx.vout.append(CTxOut(0, b"")) tx.calc_sha256() b58 = self.update_block(58, [tx]) @@ -876,7 +881,7 @@ class FullBlockTest(BitcoinTestFramework): tx.nLockTime = 0xffffffff # this locktime is non-final tx.vin.append(CTxIn(COutPoint(out[18].sha256, 0))) # don't set nSequence tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) - assert tx.vin[0].nSequence < 0xffffffff + assert_greater_than(SEQUENCE_FINAL, tx.vin[0].nSequence) tx.calc_sha256() b62 = self.update_block(62, [tx]) self.send_blocks([b62], success=False, reject_reason='bad-txns-nonfinal', reconnect=True) @@ -1024,7 +1029,7 @@ class FullBlockTest(BitcoinTestFramework): bogus_tx = CTransaction() bogus_tx.sha256 = uint256_from_str(b"23c70ed7c0506e9178fc1a987f40a33946d4ad4c962b5ae3a52546da53af0c5c") tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", 0xffffffff)) + tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", SEQUENCE_FINAL)) tx.vout.append(CTxOut(1, b"")) b70 = self.update_block(70, [tx]) self.send_blocks([b70], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True) diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index eb90b2c598..7fd0d0140b 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -13,6 +13,7 @@ from test_framework.blocktools import ( ) from test_framework.messages import ( CTransaction, + SEQUENCE_FINAL, msg_block, ) from test_framework.p2p import P2PInterface @@ -53,7 +54,7 @@ def cltv_invalidate(tx, failure_reason): # 3) the lock-time type (height vs. timestamp) of the top stack item and the # nLockTime field are not the same # 4) the top stack item is greater than the transaction's nLockTime field - # 5) the nSequence field of the txin is 0xffffffff + # 5) the nSequence field of the txin is 0xffffffff (SEQUENCE_FINAL) assert failure_reason in range(5) scheme = [ # | Script to prepend to scriptSig | nSequence | nLockTime | @@ -62,7 +63,7 @@ def cltv_invalidate(tx, failure_reason): [[OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP], None, None], [[CScriptNum(100), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 1296688602], # timestamp of genesis block [[CScriptNum(100), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 50], - [[CScriptNum(50), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0xffffffff, 50], + [[CScriptNum(50), OP_CHECKLOCKTIMEVERIFY, OP_DROP], SEQUENCE_FINAL, 50], ][failure_reason] cltv_modify_tx(tx, prepend_scriptsig=scheme[0], nsequence=scheme[1], nlocktime=scheme[2]) @@ -114,7 +115,7 @@ class BIP65Test(BitcoinTestFramework): # create one invalid tx per CLTV failure reason (5 in total) and collect them invalid_cltv_txs = [] for i in range(5): - spendtx = wallet.create_self_transfer(from_node=self.nodes[0])['tx'] + spendtx = wallet.create_self_transfer()['tx'] cltv_invalidate(spendtx, i) invalid_cltv_txs.append(spendtx) @@ -145,7 +146,7 @@ class BIP65Test(BitcoinTestFramework): # create and test one invalid tx per CLTV failure reason (5 in total) for i in range(5): - spendtx = wallet.create_self_transfer(from_node=self.nodes[0])['tx'] + spendtx = wallet.create_self_transfer()['tx'] cltv_invalidate(spendtx, i) expected_cltv_reject_reason = [ diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index c200445e81..6470c1c5eb 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -104,7 +104,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def create_self_transfer_from_utxo(self, input_tx): utxo = self.miniwallet.get_utxo(txid=input_tx.rehash(), mark_as_spent=False) - tx = self.miniwallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo)['tx'] + tx = self.miniwallet.create_self_transfer(utxo_to_spend=utxo)['tx'] return tx def create_bip112special(self, input, txversion): diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index b7cb32c842..f35ce7e0c9 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -57,7 +57,7 @@ class BIP66Test(BitcoinTestFramework): def create_tx(self, input_txid): utxo_to_spend = self.miniwallet.get_utxo(txid=input_txid, mark_as_spent=False) - return self.miniwallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_to_spend)['tx'] + return self.miniwallet.create_self_transfer(utxo_to_spend=utxo_to_spend)['tx'] def test_dersig_info(self, *, is_active): assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip66'], diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 46d5bcf1a6..233ffd60da 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -17,7 +17,6 @@ from test_framework.messages import ( from test_framework.script import ( CScript, OP_1, - OP_2, OP_DROP, OP_TRUE, ) @@ -36,16 +35,14 @@ from test_framework.util import ( # Construct 2 trivial P2SH's and the ScriptSigs that spend them # So we can create many transactions without needing to spend # time signing. -REDEEM_SCRIPT_1 = CScript([OP_1, OP_DROP]) -REDEEM_SCRIPT_2 = CScript([OP_2, OP_DROP]) -P2SH_1 = script_to_p2sh_script(REDEEM_SCRIPT_1) -P2SH_2 = script_to_p2sh_script(REDEEM_SCRIPT_2) +SCRIPT = CScript([OP_1, OP_DROP]) +P2SH = script_to_p2sh_script(SCRIPT) +REDEEM_SCRIPT = CScript([OP_TRUE, SCRIPT]) -# Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 -SCRIPT_SIG = [CScript([OP_TRUE, REDEEM_SCRIPT_1]), CScript([OP_TRUE, REDEEM_SCRIPT_2])] - -def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment): +def small_txpuzzle_randfee( + from_node, conflist, unconflist, amount, min_fee, fee_increment +): """Create and send a transaction with a random fee. The transaction pays to a trivial P2SH script, and assumes that its inputs @@ -66,20 +63,15 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) + tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + while total_in <= (amount + fee) and len(unconflist) > 0: + t = unconflist.pop(0) + total_in += t["amount"] + tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) if total_in <= amount + fee: - while total_in <= (amount + fee) and len(unconflist) > 0: - t = unconflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) - if total_in <= amount + fee: - raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") - tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH_1)) - tx.vout.append(CTxOut(int(amount * COIN), P2SH_2)) - # These transactions don't need to be signed, but we still have to insert - # the ScriptSig that will satisfy the ScriptPubKey. - for inp in tx.vin: - inp.scriptSig = SCRIPT_SIG[inp.prevout.n] + raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") + tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH)) + tx.vout.append(CTxOut(int(amount * COIN), P2SH)) txid = from_node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) unconflist.append({"txid": txid, "vout": 1, "amount": amount}) @@ -87,34 +79,6 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee return (tx.serialize().hex(), fee) -def split_inputs(from_node, txins, txouts, initial_split=False): - """Generate a lot of inputs so we can generate a ton of transactions. - - This function takes an input from txins, and creates and sends a transaction - which splits the value into 2 outputs which are appended to txouts. - Previously this was designed to be small inputs so they wouldn't have - a high coin age when the notion of priority still existed.""" - - prevtxout = txins.pop() - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(int(prevtxout["txid"], 16), prevtxout["vout"]), b"")) - - half_change = satoshi_round(prevtxout["amount"] / 2) - rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000") - tx.vout.append(CTxOut(int(half_change * COIN), P2SH_1)) - tx.vout.append(CTxOut(int(rem_change * COIN), P2SH_2)) - - # If this is the initial split we actually need to sign the transaction - # Otherwise we just need to insert the proper ScriptSig - if (initial_split): - completetx = from_node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] - else: - tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]] - completetx = tx.serialize().hex() - txid = from_node.sendrawtransaction(hexstring=completetx, maxfeerate=0) - txouts.append({"txid": txid, "vout": 0, "amount": half_change}) - txouts.append({"txid": txid, "vout": 1, "amount": rem_change}) - def check_raw_estimates(node, fees_seen): """Call estimaterawfee and verify that the estimates meet certain invariants.""" @@ -125,7 +89,10 @@ def check_raw_estimates(node, fees_seen): assert_greater_than(feerate, 0) if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): - raise AssertionError(f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})") + raise AssertionError( + f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" + ) + def check_smart_estimates(node, fees_seen): """Call estimatesmartfee and verify that the estimates meet certain invariants.""" @@ -133,8 +100,8 @@ def check_smart_estimates(node, fees_seen): delta = 1.0e-6 # account for rounding error last_feerate = float(max(fees_seen)) all_smart_estimates = [node.estimatesmartfee(i) for i in range(1, 26)] - mempoolMinFee = node.getmempoolinfo()['mempoolminfee'] - minRelaytxFee = node.getmempoolinfo()['minrelaytxfee'] + mempoolMinFee = node.getmempoolinfo()["mempoolminfee"] + minRelaytxFee = node.getmempoolinfo()["minrelaytxfee"] for i, e in enumerate(all_smart_estimates): # estimate is for i+1 feerate = float(e["feerate"]) assert_greater_than(feerate, 0) @@ -142,9 +109,13 @@ def check_smart_estimates(node, fees_seen): assert_greater_than_or_equal(feerate, float(minRelaytxFee)) if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): - raise AssertionError(f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})") + raise AssertionError( + f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" + ) if feerate - delta > last_feerate: - raise AssertionError(f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms") + raise AssertionError( + f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms" + ) last_feerate = feerate if i == 0: @@ -152,6 +123,7 @@ def check_smart_estimates(node, fees_seen): else: assert_greater_than_or_equal(i + 1, e["blocks"]) + def check_estimates(node, fees_seen): check_raw_estimates(node, fees_seen) check_smart_estimates(node, fees_seen) @@ -159,27 +131,25 @@ def check_estimates(node, fees_seen): def send_tx(node, utxo, feerate): """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb).""" - overhead, op, scriptsig, nseq, value, spk = 10, 36, 5, 4, 8, 24 - tx_size = overhead + op + scriptsig + nseq + value + spk - fee = tx_size * feerate - tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), SCRIPT_SIG[utxo["vout"]])] - tx.vout = [CTxOut(int(utxo["amount"] * COIN) - fee, P2SH_1)] - txid = node.sendrawtransaction(tx.serialize().hex()) + tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), REDEEM_SCRIPT)] + tx.vout = [CTxOut(int(utxo["amount"] * COIN), P2SH)] + + # vbytes == bytes as we are using legacy transactions + fee = tx.get_vsize() * feerate + tx.vout[0].nValue -= fee - return txid + return node.sendrawtransaction(tx.serialize().hex()) class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 - # mine non-standard txs (e.g. txs with "dust" outputs) # Force fSendTrickle to true (via whitelist.noban) self.extra_args = [ - ["-acceptnonstdtxn", "-whitelist=noban@127.0.0.1"], - ["-acceptnonstdtxn", "-whitelist=noban@127.0.0.1", "-blockmaxweight=68000"], - ["-acceptnonstdtxn", "-whitelist=noban@127.0.0.1", "-blockmaxweight=32000"], + ["-whitelist=noban@127.0.0.1"], + ["-whitelist=noban@127.0.0.1", "-blockmaxweight=68000"], + ["-whitelist=noban@127.0.0.1", "-blockmaxweight=32000"], ] def skip_test_if_missing_module(self): @@ -212,11 +182,17 @@ class EstimateFeeTest(BitcoinTestFramework): random.shuffle(self.confutxo) for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) - (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo, - self.memutxo, Decimal("0.005"), min_fee, min_fee) + (txhex, fee) = small_txpuzzle_randfee( + self.nodes[from_index], + self.confutxo, + self.memutxo, + Decimal("0.005"), + min_fee, + min_fee, + ) tx_kbytes = (len(txhex) // 2) / 1000.0 self.fees_per_kb.append(float(fee) / tx_kbytes) - self.sync_mempools(wait=.1) + self.sync_mempools(wait=0.1) mined = mining_node.getblock(self.generate(mining_node, 1)[0], True)["tx"] # update which txouts are confirmed newmem = [] @@ -229,46 +205,45 @@ class EstimateFeeTest(BitcoinTestFramework): def initial_split(self, node): """Split two coinbase UTxOs into many small coins""" - self.txouts = [] - self.txouts2 = [] - # Split a coinbase into two transaction puzzle outputs - split_inputs(node, node.listunspent(0), self.txouts, True) - - # Mine + utxo_count = 2048 + self.confutxo = [] + splitted_amount = Decimal("0.04") + fee = Decimal("0.1") + change = Decimal("100") - splitted_amount * utxo_count - fee + tx = CTransaction() + tx.vin = [ + CTxIn(COutPoint(int(cb["txid"], 16), cb["vout"])) + for cb in node.listunspent()[:2] + ] + tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)] + tx.vout.append(CTxOut(int(change * COIN), P2SH)) + txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] + txid = node.sendrawtransaction(txhex) + self.confutxo = [ + {"txid": txid, "vout": i, "amount": splitted_amount} + for i in range(utxo_count) + ] while len(node.getrawmempool()) > 0: self.generate(node, 1, sync_fun=self.no_op) - # Repeatedly split those 2 outputs, doubling twice for each rep - # Use txouts to monitor the available utxo, since these won't be tracked in wallet - reps = 0 - while reps < 5: - # Double txouts to txouts2 - while len(self.txouts) > 0: - split_inputs(node, self.txouts, self.txouts2) - while len(node.getrawmempool()) > 0: - self.generate(node, 1, sync_fun=self.no_op) - # Double txouts2 to txouts - while len(self.txouts2) > 0: - split_inputs(node, self.txouts2, self.txouts) - while len(node.getrawmempool()) > 0: - self.generate(node, 1, sync_fun=self.no_op) - reps += 1 - def sanity_check_estimates_range(self): """Populate estimation buckets, assert estimates are in a sane range and are strictly increasing as the target decreases.""" self.fees_per_kb = [] self.memutxo = [] - self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting self.log.info("Will output estimates for 1/2/3/6/15/25 blocks") for _ in range(2): - self.log.info("Creating transactions and mining them with a block size that can't keep up") + self.log.info( + "Creating transactions and mining them with a block size that can't keep up" + ) # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine self.transact_and_mine(10, self.nodes[2]) check_estimates(self.nodes[1], self.fees_per_kb) - self.log.info("Creating transactions and mining them at a block size that is just big enough") + self.log.info( + "Creating transactions and mining them at a block size that is just big enough" + ) # Generate transactions while mining 10 more blocks, this time with node1 # which mines blocks with capacity just above the rate that transactions are being created self.transact_and_mine(10, self.nodes[1]) @@ -277,12 +252,13 @@ class EstimateFeeTest(BitcoinTestFramework): # Finish by mining a normal-sized block: while len(self.nodes[1].getrawmempool()) > 0: self.generate(self.nodes[1], 1) + self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) def test_feerate_mempoolminfee(self): - high_val = 3*self.nodes[1].estimatesmartfee(1)['feerate'] - self.restart_node(1, extra_args=[f'-minrelaytxfee={high_val}']) + high_val = 3 * self.nodes[1].estimatesmartfee(1)["feerate"] + self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}"]) check_estimates(self.nodes[1], self.fees_per_kb) self.restart_node(1) @@ -303,7 +279,7 @@ class EstimateFeeTest(BitcoinTestFramework): utxos_to_respend = [] txids_to_replace = [] - assert len(utxos) >= 250 + assert_greater_than_or_equal(len(utxos), 250) for _ in range(5): # Broadcast 45 low fee transactions that will need to be RBF'd for _ in range(45): @@ -315,27 +291,24 @@ class EstimateFeeTest(BitcoinTestFramework): for _ in range(5): send_tx(node, utxos.pop(0), low_feerate) # Mine the transactions on another node - self.sync_mempools(wait=.1, nodes=[node, miner]) + self.sync_mempools(wait=0.1, nodes=[node, miner]) for txid in txids_to_replace: miner.prioritisetransaction(txid=txid, fee_delta=-COIN) self.generate(miner, 1) # RBF the low-fee transactions - while True: - try: - u = utxos_to_respend.pop(0) - send_tx(node, u, high_feerate) - except IndexError: - break + while len(utxos_to_respend) > 0: + u = utxos_to_respend.pop(0) + send_tx(node, u, high_feerate) # Mine the last replacement txs - self.sync_mempools(wait=.1, nodes=[node, miner]) + self.sync_mempools(wait=0.1, nodes=[node, miner]) self.generate(miner, 1) # Only 10% of the transactions were really confirmed with a low feerate, # the rest needed to be RBF'd. We must return the 90% conf rate feerate. - high_feerate_kvb = Decimal(high_feerate) / COIN * 10**3 + high_feerate_kvb = Decimal(high_feerate) / COIN * 10 ** 3 est_feerate = node.estimatesmartfee(2)["feerate"] - assert est_feerate == high_feerate_kvb + assert_equal(est_feerate, high_feerate_kvb) def run_test(self): self.log.info("This test is time consuming, please be patient") @@ -359,7 +332,9 @@ class EstimateFeeTest(BitcoinTestFramework): self.sanity_check_estimates_range() # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee - self.log.info("Test fee rate estimation after restarting node with high MempoolMinFee") + self.log.info( + "Test fee rate estimation after restarting node with high MempoolMinFee" + ) self.test_feerate_mempoolminfee() self.log.info("Restarting node with fresh estimation") @@ -375,9 +350,10 @@ class EstimateFeeTest(BitcoinTestFramework): self.log.info("Testing that fee estimation is disabled in blocksonly.") self.restart_node(0, ["-blocksonly"]) - assert_raises_rpc_error(-32603, "Fee estimation disabled", - self.nodes[0].estimatesmartfee, 2) + assert_raises_rpc_error( + -32603, "Fee estimation disabled", self.nodes[0].estimatesmartfee, 2 + ) -if __name__ == '__main__': +if __name__ == "__main__": EstimateFeeTest().main() diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 16815c1afc..4b56b0c26b 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -93,7 +93,7 @@ class InitStressTest(BitcoinTestFramework): additional_lines = random.randint(1, num_total_logs) self.log.debug(f"Starting node and will exit after {additional_lines} lines") node.start(extra_args=['-txindex=1']) - logfile = open(node.debug_log_path, 'r', encoding='utf8') + logfile = open(node.debug_log_path, 'rb') MAX_SECS_TO_WAIT = 10 start = time.time() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 39cdb2750b..ba3c5053cb 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -96,9 +96,6 @@ class PruneTest(BitcoinTestFramework): ] self.rpc_timeout = 120 - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): self.setup_nodes() @@ -114,7 +111,8 @@ class PruneTest(BitcoinTestFramework): def setup_nodes(self): self.add_nodes(self.num_nodes, self.extra_args) self.start_nodes() - self.import_deterministic_coinbase_privkeys() + if self.is_wallet_compiled(): + self.import_deterministic_coinbase_privkeys() def create_big_chain(self): # Start by creating some coinbases we can spend later @@ -474,8 +472,9 @@ class PruneTest(BitcoinTestFramework): self.log.info("Test manual pruning with timestamps") self.manual_test(4, use_timestamp=True) - self.log.info("Test wallet re-scan") - self.wallet_test() + if self.is_wallet_compiled(): + self.log.info("Test wallet re-scan") + self.wallet_test() self.log.info("Test invalid pruning command line options") self.test_invalid_command_line_options() diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 5722e71c7a..f0ed914461 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -14,6 +14,7 @@ from test_framework.messages import ( CTransaction, CTxIn, CTxOut, + SEQUENCE_FINAL, ) from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework @@ -114,7 +115,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): """Simple doublespend""" # we use MiniWallet to create a transaction template with inputs correctly set, # and modify the output (amount, scriptPubKey) according to our needs - tx_template = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] + tx_template = self.wallet.create_self_transfer()['tx'] tx1a = deepcopy(tx_template) tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] @@ -402,7 +403,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Create a non-opting in transaction tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0xffffffff)] + tx1a.vin = [CTxIn(tx0_outpoint, nSequence=SEQUENCE_FINAL)] tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] tx1a_hex = tx1a.serialize().hex() tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) @@ -445,7 +446,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx2a_txid = int(tx2a_txid, 16) tx3a = CTransaction() - tx3a.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0xffffffff), + tx3a.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=SEQUENCE_FINAL), CTxIn(COutPoint(tx2a_txid, 0), nSequence=0xfffffffd)] tx3a.vout = [CTxOut(int(0.9 * COIN), CScript([b'c'])), CTxOut(int(0.9 * COIN), CScript([b'd']))] tx3a_hex = tx3a.serialize().hex() @@ -562,7 +563,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(True, self.nodes[0].getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) replacement_parent_tx = self.wallet.create_self_transfer( - from_node=self.nodes[0], utxo_to_spend=confirmed_utxo, sequence=BIP125_SEQUENCE_NUMBER, fee_rate=Decimal('0.02'), @@ -579,7 +579,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): optout_child_tx = self.wallet.send_self_transfer( from_node=self.nodes[0], utxo_to_spend=parent_utxo, - sequence=0xffffffff, + sequence=SEQUENCE_FINAL, fee_rate=Decimal('0.01'), ) @@ -587,9 +587,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(True, self.nodes[0].getmempoolentry(optout_child_tx['txid'])['bip125-replaceable']) replacement_child_tx = self.wallet.create_self_transfer( - from_node=self.nodes[0], utxo_to_spend=parent_utxo, - sequence=0xffffffff, + sequence=SEQUENCE_FINAL, fee_rate=Decimal('0.02'), mempool_valid=False, ) @@ -608,7 +607,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): replacement_parent_tx = self.wallet.send_self_transfer( from_node=self.nodes[0], utxo_to_spend=confirmed_utxo, - sequence=0xffffffff, + sequence=SEQUENCE_FINAL, fee_rate=Decimal('0.03'), ) # Check that child is removed and update wallet utxo state diff --git a/test/functional/feature_startupnotify.py b/test/functional/feature_startupnotify.py index 71135dcd3b..c6aa837768 100755 --- a/test/functional/feature_startupnotify.py +++ b/test/functional/feature_startupnotify.py @@ -26,7 +26,7 @@ class StartupNotifyTest(BitcoinTestFramework): self.log.info("Test -startupnotify command is run when node starts") self.restart_node(0, extra_args=[f"-startupnotify=echo '{FILE_NAME}' >> {NODE_DIR}/{FILE_NAME}"]) - assert os.path.exists(tmpdir_file) + self.wait_until(lambda: os.path.exists(tmpdir_file)) self.log.info("Test -startupnotify is executed once") with open(tmpdir_file, "r", encoding="utf8") as f: diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index e9da6edaf6..3e3d4b3c77 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -19,6 +19,7 @@ from test_framework.messages import ( CTxIn, CTxInWitness, CTxOut, + SEQUENCE_FINAL, ) from test_framework.script import ( ANNEX_TAG, @@ -1516,7 +1517,7 @@ class TaprootTest(BitcoinTestFramework): assert self.nodes[1].getblockcount() == 0 coinbase = CTransaction() coinbase.nVersion = 1 - coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), 0xffffffff)] + coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), SEQUENCE_FINAL)] coinbase.vout = [CTxOut(5000000000, CScript([OP_1]))] coinbase.nLockTime = 0 coinbase.rehash() @@ -1604,7 +1605,7 @@ class TaprootTest(BitcoinTestFramework): val = 42000000 * (i + 7) tx = CTransaction() tx.nVersion = 1 - tx.vin = [CTxIn(COutPoint(lasttxid, i & 1), CScript([]), 0xffffffff)] + tx.vin = [CTxIn(COutPoint(lasttxid, i & 1), CScript([]), SEQUENCE_FINAL)] tx.vout = [CTxOut(val, spk), CTxOut(amount - val, CScript([OP_1]))] if i & 1: tx.vout = list(reversed(tx.vout)) @@ -1664,7 +1665,7 @@ class TaprootTest(BitcoinTestFramework): tx.vin = [] inputs = [] input_spks = [tap_spks[0], tap_spks[1], old_spks[0], tap_spks[2], tap_spks[5], old_spks[2], tap_spks[6], tap_spks[3], tap_spks[4]] - sequences = [0, 0xffffffff, 0xffffffff, 0xfffffffe, 0xfffffffe, 0, 0, 0xffffffff, 0xffffffff] + sequences = [0, SEQUENCE_FINAL, SEQUENCE_FINAL, 0xfffffffe, 0xfffffffe, 0, 0, SEQUENCE_FINAL, SEQUENCE_FINAL] hashtypes = [SIGHASH_SINGLE, SIGHASH_SINGLE|SIGHASH_ANYONECANPAY, SIGHASH_ALL, SIGHASH_ALL, SIGHASH_DEFAULT, SIGHASH_ALL, SIGHASH_NONE, SIGHASH_NONE|SIGHASH_ANYONECANPAY, SIGHASH_ALL|SIGHASH_ANYONECANPAY] for i, spk in enumerate(input_spks): tx.vin.append(CTxIn(spend_info[spk]['prevout'], CScript(), sequences[i])) diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index 33b7615aea..75180e62a2 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -69,8 +69,8 @@ class UTXOSetHashTest(BitcoinTestFramework): assert_equal(finalized[::-1].hex(), node_muhash) self.log.info("Test deterministic UTXO set hash results") - assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "221f245cf4c9010eeb7f5183d342c002ae6c1c27e98aa357dccb788c21d98049") - assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "7c0890c68501f7630d36aeb3999dc924e63af084ae1bbfba11dd462144637635") + assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "3a570529b4c32e77268de1f81b903c75cc2da53c48df0d125c1e697ba7c8c7b7") + assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "a13e0e70eb8acc786549596e3bc154623f1a5a622ba2f70715f6773ec745f435") def run_test(self): self.test_muhash_implementation() diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 2842b2534d..06aa5608bb 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -22,6 +22,10 @@ from test_framework.util import ( from test_framework.messages import BLOCK_HEADER_SIZE +INVALID_PARAM = "abc" +UNKNOWN_PARAM = "0000000000000000000000000000000000000000000000000000000000000000" + + class ReqType(Enum): JSON = 1 BIN = 2 @@ -103,6 +107,12 @@ class RESTTest (BitcoinTestFramework): n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1')) spending = (txid, n) + # Test /tx with an invalid and an unknown txid + resp = self.test_rest_request(uri=f"/tx/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") + resp = self.test_rest_request(uri=f"/tx/{UNKNOWN_PARAM}", ret_type=RetType.OBJ, status=404) + assert_equal(resp.read().decode('utf-8').rstrip(), f"{UNKNOWN_PARAM} not found") + self.log.info("Query an unspent TXO using the /getutxos URI") self.generatetoaddress(self.nodes[1], 1, not_related_address) @@ -205,8 +215,8 @@ class RESTTest (BitcoinTestFramework): bb_hash = self.nodes[0].getbestblockhash() # Check result if block does not exists - assert_equal(self.test_rest_request('/headers/1/0000000000000000000000000000000000000000000000000000000000000000'), []) - self.test_rest_request('/block/0000000000000000000000000000000000000000000000000000000000000000', status=404, ret_type=RetType.OBJ) + assert_equal(self.test_rest_request(f"/headers/1/{UNKNOWN_PARAM}"), []) + self.test_rest_request(f"/block/{UNKNOWN_PARAM}", status=404, ret_type=RetType.OBJ) # Check result if block is not in the active chain self.nodes[0].invalidateblock(bb_hash) @@ -250,8 +260,8 @@ class RESTTest (BitcoinTestFramework): assert_equal(blockhash, bb_hash) # Check invalid blockhashbyheight requests - resp = self.test_rest_request("/blockhashbyheight/abc", ret_type=RetType.OBJ, status=400) - assert_equal(resp.read().decode('utf-8').rstrip(), "Invalid height: abc") + resp = self.test_rest_request(f"/blockhashbyheight/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid height: {INVALID_PARAM}") resp = self.test_rest_request("/blockhashbyheight/1000000", ret_type=RetType.OBJ, status=404) assert_equal(resp.read().decode('utf-8').rstrip(), "Block height out of range") resp = self.test_rest_request("/blockhashbyheight/-1", ret_type=RetType.OBJ, status=400) diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 44db8bb00a..d3961e753d 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" +from copy import deepcopy from decimal import Decimal import math @@ -17,6 +18,7 @@ from test_framework.messages import ( CTxOut, MAX_BLOCK_WEIGHT, MAX_MONEY, + SEQUENCE_FINAL, tx_from_hex, ) from test_framework.script import ( @@ -33,6 +35,7 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet class MempoolAcceptanceTest(BitcoinTestFramework): @@ -43,9 +46,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ]] * self.num_nodes self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def check_mempool_result(self, result_expected, *args, **kwargs): """Wrapper to check result of testmempoolaccept on node_0's mempool""" result_test = self.nodes[0].testmempoolaccept(*args, **kwargs) @@ -56,12 +56,13 @@ class MempoolAcceptanceTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] + self.wallet = MiniWallet(node) + self.wallet.rescan_utxos() self.log.info('Start with empty mempool, and 200 blocks') self.mempool_size = 0 assert_equal(node.getblockcount(), 200) assert_equal(node.getmempoolinfo()['size'], self.mempool_size) - coins = node.listunspent() self.log.info('Should not accept garbage to testmempoolaccept') assert_raises_rpc_error(-3, 'Expected type array, got string', lambda: node.testmempoolaccept(rawtxs='ff00baar')) @@ -70,12 +71,12 @@ class MempoolAcceptanceTest(BitcoinTestFramework): assert_raises_rpc_error(-22, 'TX decode failed', lambda: node.testmempoolaccept(rawtxs=['ff00baar'])) self.log.info('A transaction already in the blockchain') - coin = coins.pop() # Pick a random coin(base) to spend - raw_tx_in_block = node.signrawtransactionwithwallet(node.createrawtransaction( - inputs=[{'txid': coin['txid'], 'vout': coin['vout']}], - outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}], - ))['hex'] - txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, maxfeerate=0) + tx = self.wallet.create_self_transfer()['tx'] # Pick a random coin(base) to spend + tx.vout.append(deepcopy(tx.vout[0])) + tx.vout[0].nValue = int(0.3 * COIN) + tx.vout[1].nValue = int(49 * COIN) + raw_tx_in_block = tx.serialize().hex() + txid_in_block = self.wallet.sendrawtransaction(from_node=node, tx_hex=raw_tx_in_block, maxfeerate=0) self.generate(node, 1) self.mempool_size = 0 self.check_mempool_result( @@ -85,11 +86,10 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction not in the mempool') fee = Decimal('0.000007') - raw_tx_0 = node.signrawtransactionwithwallet(node.createrawtransaction( - inputs=[{"txid": txid_in_block, "vout": 0, "sequence": BIP125_SEQUENCE_NUMBER}], # RBF is used later - outputs=[{node.getnewaddress(): Decimal('0.3') - fee}], - ))['hex'] - tx = tx_from_hex(raw_tx_0) + utxo_to_spend = self.wallet.get_utxo(txid=txid_in_block) # use 0.3 BTC UTXO + tx = self.wallet.create_self_transfer(utxo_to_spend=utxo_to_spend, sequence=BIP125_SEQUENCE_NUMBER)['tx'] + tx.vout[0].nValue = int((Decimal('0.3') - fee) * COIN) + raw_tx_0 = tx.serialize().hex() txid_0 = tx.rehash() self.check_mempool_result( result_expected=[{'txid': txid_0, 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': fee}}], @@ -97,15 +97,15 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ) self.log.info('A final transaction not in the mempool') - coin = coins.pop() # Pick a random coin(base) to spend output_amount = Decimal('0.025') - raw_tx_final = node.signrawtransactionwithwallet(node.createrawtransaction( - inputs=[{'txid': coin['txid'], 'vout': coin['vout'], "sequence": 0xffffffff}], # SEQUENCE_FINAL - outputs=[{node.getnewaddress(): output_amount}], + tx = self.wallet.create_self_transfer( + sequence=SEQUENCE_FINAL, locktime=node.getblockcount() + 2000, # Can be anything - ))['hex'] + )['tx'] + tx.vout[0].nValue = int(output_amount * COIN) + raw_tx_final = tx.serialize().hex() tx = tx_from_hex(raw_tx_final) - fee_expected = coin['amount'] - output_amount + fee_expected = Decimal('50.0') - output_amount self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': fee_expected}}], rawtxs=[tx.serialize().hex()], @@ -126,8 +126,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx = tx_from_hex(raw_tx_0) tx.vout[0].nValue -= int(fee * COIN) # Double the fee tx.vin[0].nSequence = BIP125_SEQUENCE_NUMBER + 1 # Now, opt out of RBF - raw_tx_0 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] - tx = tx_from_hex(raw_tx_0) + raw_tx_0 = tx.serialize().hex() txid_0 = tx.rehash() self.check_mempool_result( result_expected=[{'txid': txid_0, 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': (2 * fee)}}], @@ -140,7 +139,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # take original raw_tx_0 tx = tx_from_hex(raw_tx_0) tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee - # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'txn-mempool-conflict'}], rawtxs=[tx.serialize().hex()], @@ -150,7 +148,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction with missing inputs, that never existed') tx = tx_from_hex(raw_tx_0) tx.vin[0].prevout = COutPoint(hash=int('ff' * 32, 16), n=14) - # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'missing-inputs'}], rawtxs=[tx.serialize().hex()], @@ -159,17 +156,17 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction with missing inputs, that existed once in the past') tx = tx_from_hex(raw_tx_0) tx.vin[0].prevout.n = 1 # Set vout to 1, to spend the other outpoint (49 coins) of the in-chain-tx we want to double spend - raw_tx_1 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] + raw_tx_1 = tx.serialize().hex() txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, maxfeerate=0) # Now spend both to "clearly hide" the outputs, ie. remove the coins from the utxo set by spending them - raw_tx_spend_both = node.signrawtransactionwithwallet(node.createrawtransaction( - inputs=[ - {'txid': txid_0, 'vout': 0}, - {'txid': txid_1, 'vout': 0}, - ], - outputs=[{node.getnewaddress(): 0.1}] - ))['hex'] - txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, maxfeerate=0) + tx = self.wallet.create_self_transfer()['tx'] + tx.vin.append(deepcopy(tx.vin[0])) + tx.wit.vtxinwit.append(deepcopy(tx.wit.vtxinwit[0])) + tx.vin[0].prevout = COutPoint(hash=int(txid_0, 16), n=0) + tx.vin[1].prevout = COutPoint(hash=int(txid_1, 16), n=0) + tx.vout[0].nValue = int(0.1 * COIN) + raw_tx_spend_both = tx.serialize().hex() + txid_spend_both = self.wallet.sendrawtransaction(from_node=node, tx_hex=raw_tx_spend_both, maxfeerate=0) self.generate(node, 1) self.mempool_size = 0 # Now see if we can add the coins back to the utxo set by sending the exact txs again @@ -182,12 +179,11 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[raw_tx_1], ) - self.log.info('Create a signed "reference" tx for later use') - raw_tx_reference = node.signrawtransactionwithwallet(node.createrawtransaction( - inputs=[{'txid': txid_spend_both, 'vout': 0}], - outputs=[{node.getnewaddress(): 0.05}], - ))['hex'] - tx = tx_from_hex(raw_tx_reference) + self.log.info('Create a "reference" tx for later use') + utxo_to_spend = self.wallet.get_utxo(txid=txid_spend_both) + tx = self.wallet.create_self_transfer(utxo_to_spend=utxo_to_spend, sequence=SEQUENCE_FINAL)['tx'] + tx.vout[0].nValue = int(0.05 * COIN) + raw_tx_reference = tx.serialize().hex() # Reference tx should be valid on itself self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True, 'vsize': tx.get_vsize(), 'fees': { 'base': Decimal('0.1') - Decimal('0.05')}}], @@ -198,8 +194,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction with no outputs') tx = tx_from_hex(raw_tx_reference) tx.vout = [] - # Skip re-signing the transaction for context independent checks from now on - # tx = tx_from_hex(node.signrawtransactionwithwallet(tx.serialize().hex())['hex']) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-vout-empty'}], rawtxs=[tx.serialize().hex()], @@ -256,7 +250,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ) self.log.info('A coinbase transaction') - # Pick the input of the first tx we signed, so it has to be a coinbase tx + # Pick the input of the first tx we created, so it has to be a coinbase tx raw_tx_coinbase_spent = node.getrawtransaction(txid=node.decoderawtransaction(hexstring=raw_tx_in_block)['vin'][0]['txid']) tx = tx_from_hex(raw_tx_coinbase_spent) self.check_mempool_result( @@ -333,7 +327,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction that is locked by BIP68 sequence logic') tx = tx_from_hex(raw_tx_reference) tx.vin[0].nSequence = 2 # We could include it in the second block mined from now, but not the very next one - # Can skip re-signing the tx because of early rejection self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'non-BIP68-final'}], rawtxs=[tx.serialize().hex()], diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 7e940fa3ca..91f2d0051a 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -45,14 +45,13 @@ class MempoolCoinbaseTest(BitcoinTestFramework): utxo_2 = wallet.get_utxo(txid=coinbase_txids[2]) utxo_3 = wallet.get_utxo(txid=coinbase_txids[3]) self.log.info("Create three transactions spending from coinbase utxos: spend_1, spend_2, spend_3") - spend_1 = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_1) - spend_2 = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_2) - spend_3 = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_3) + spend_1 = wallet.create_self_transfer(utxo_to_spend=utxo_1) + spend_2 = wallet.create_self_transfer(utxo_to_spend=utxo_2) + spend_3 = wallet.create_self_transfer(utxo_to_spend=utxo_3) self.log.info("Create another transaction which is time-locked to two blocks in the future") utxo = wallet.get_utxo(txid=coinbase_txids[0]) timelock_tx = wallet.create_self_transfer( - from_node=self.nodes[0], utxo_to_spend=utxo, mempool_valid=False, locktime=self.nodes[0].getblockcount() + 2 @@ -71,9 +70,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework): self.log.info("Create spend_2_1 and spend_3_1") spend_2_utxo = wallet.get_utxo(txid=spend_2['txid']) - spend_2_1 = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=spend_2_utxo) + spend_2_1 = wallet.create_self_transfer(utxo_to_spend=spend_2_utxo) spend_3_utxo = wallet.get_utxo(txid=spend_3['txid']) - spend_3_1 = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=spend_3_utxo) + spend_3_1 = wallet.create_self_transfer(utxo_to_spend=spend_3_utxo) self.log.info("Broadcast and mine spend_3_1") spend_3_1_id = self.nodes[0].sendrawtransaction(spend_3_1['hex']) diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index 5afa6be925..9c43ddaf6f 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -40,7 +40,7 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): spend_mature_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_mature)["txid"] # other coinbase should be too immature to spend - immature_tx = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_immature, mempool_valid=False) + immature_tx = wallet.create_self_transfer(utxo_to_spend=utxo_immature, mempool_valid=False) assert_raises_rpc_error(-26, "bad-txns-premature-spend-of-coinbase", lambda: self.nodes[0].sendrawtransaction(immature_tx['hex'])) diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 16c15e3f74..51de582ce0 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -17,7 +17,7 @@ from test_framework.util import assert_equal class MempoolUpdateFromBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000']] + self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000', '-limitancestorcount=100']] def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index f8d002e664..9c64bb1945 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -29,6 +29,8 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + VERSIONBITS_TOP_BITS = 0x20000000 VERSIONBITS_DEPLOYMENT_TESTDUMMY_BIT = 28 @@ -51,14 +53,11 @@ class MiningTest(BitcoinTestFramework): self.setup_clean_chain = True self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def mine_chain(self): self.log.info('Create some old blocks') for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): self.nodes[0].setmocktime(t) - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) mining_info = self.nodes[0].getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['currentblocktx'], 0) @@ -75,8 +74,9 @@ class MiningTest(BitcoinTestFramework): self.connect_nodes(0, 1) def run_test(self): - self.mine_chain() node = self.nodes[0] + self.wallet = MiniWallet(node) + self.mine_chain() def assert_submitblock(block, result_str_1, result_str_2=None): block.solve() @@ -95,7 +95,7 @@ class MiningTest(BitcoinTestFramework): assert_equal(mining_info['pooledtx'], 0) self.log.info("getblocktemplate: Test default witness commitment") - txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + txid = int(self.wallet.send_self_transfer(from_node=node)['wtxid'], 16) tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS) # Check that default_witness_commitment is present. diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6e48341259..6f142f23f2 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -102,7 +102,7 @@ class P2PBlocksOnly(BitcoinTestFramework): def check_p2p_tx_violation(self): self.log.info('Check that txs from P2P are rejected and result in disconnect') - spendtx = self.miniwallet.create_self_transfer(from_node=self.nodes[0]) + spendtx = self.miniwallet.create_self_transfer() with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): self.nodes[0].p2ps[0].send_message(msg_tx(spendtx['tx'])) diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 721a5a4c57..7284ecde83 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -57,11 +57,11 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal(len(self.nodes[1].listbanned()), 0) self.log.info("setban: test persistence across node restart") - self.nodes[1].setban("127.0.0.0/32", "add") - self.nodes[1].setban("127.0.0.0/24", "add") # Set the mocktime so we can control when bans expire old_time = int(time.time()) self.nodes[1].setmocktime(old_time) + self.nodes[1].setban("127.0.0.0/32", "add") + self.nodes[1].setban("127.0.0.0/24", "add") self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds listBeforeShutdown = self.nodes[1].listbanned() diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py index 7aede0e947..7eeb745817 100755 --- a/test/functional/rpc_generateblock.py +++ b/test/functional/rpc_generateblock.py @@ -63,7 +63,7 @@ class GenerateBlockTest(BitcoinTestFramework): assert_equal(block['tx'][1], txid) self.log.info('Generate block with raw tx') - rawtx = miniwallet.create_self_transfer(from_node=node)['hex'] + rawtx = miniwallet.create_self_transfer()['hex'] hash = self.generateblock(node, address, [rawtx])['hash'] block = node.getblock(hash, 1) @@ -74,7 +74,7 @@ class GenerateBlockTest(BitcoinTestFramework): self.log.info('Fail to generate block with out of order txs') txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] utxo1 = miniwallet.get_utxo(txid=txid1) - rawtx2 = miniwallet.create_self_transfer(from_node=node, utxo_to_spend=utxo1)['hex'] + rawtx2 = miniwallet.create_self_transfer(utxo_to_spend=utxo1)['hex'] assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) self.log.info('Fail to generate block with txid not in mempool') diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index caa9b86969..40fcbf7761 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -22,6 +22,7 @@ from .messages import ( CTxIn, CTxInWitness, CTxOut, + SEQUENCE_FINAL, hash256, ser_uint256, tx_from_hex, @@ -128,7 +129,7 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValu If extra_output_script is given, make a 0-value output to that script. This is useful to pad block weight/sigops as needed. """ coinbase = CTransaction() - coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)) + coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), SEQUENCE_FINAL)) coinbaseoutput = CTxOut() coinbaseoutput.nValue = nValue * COIN if nValue == 50: @@ -156,7 +157,7 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C """ tx = CTransaction() assert n < len(prevtx.vout) - tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, 0xffffffff)) + tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, SEQUENCE_FINAL)) tx.vout.append(CTxOut(amount, script_pub_key)) tx.calc_sha256() return tx diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index e3599cad49..f57b6e7494 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -40,6 +40,7 @@ COIN = 100000000 # 1 btc in satoshis MAX_MONEY = 21000000 * COIN BIP125_SEQUENCE_NUMBER = 0xfffffffd # Sequence number that is rbf-opt-in (BIP 125) and csv-opt-out (BIP 68) +SEQUENCE_FINAL = 0xffffffff # Sequence number that disables nLockTime if set for every input of a tx MAX_PROTOCOL_MESSAGE_LENGTH = 4000000 # Maximum length of incoming protocol messages MAX_HEADERS_RESULTS = 2000 # Number of headers sent in one getheaders result @@ -508,7 +509,7 @@ class CTransaction: def __init__(self, tx=None): if tx is None: - self.nVersion = 1 + self.nVersion = 2 self.vin = [] self.vout = [] self.wit = CTxWitness() diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 9158dd0942..dd41a740ae 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -179,8 +179,9 @@ class MiniWallet: txid = self.sendrawtransaction(from_node=from_node, tx_hex=tx.serialize().hex()) return txid, 1 - def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node=None, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + from_node = from_node or self._test_node utxo_to_spend = utxo_to_spend or self.get_utxo() if self._priv_key is None: vsize = Decimal(104) # anyone-can-spend @@ -213,8 +214,8 @@ class MiniWallet: assert_equal(tx_info['fees']['base'], utxo_to_spend['value'] - Decimal(send_value) / COIN) return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} - def sendrawtransaction(self, *, from_node, tx_hex): - txid = from_node.sendrawtransaction(tx_hex) + def sendrawtransaction(self, *, from_node, tx_hex, **kwargs): + txid = from_node.sendrawtransaction(hexstring=tx_hex, **kwargs) self.scan_tx(from_node.decoderawtransaction(tx_hex)) return txid diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index df9aec6281..eb2d030f4a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -530,33 +530,35 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= max_len_name = len(max(test_list, key=len)) test_count = len(test_list) - for i in range(test_count): - test_result, testdir, stdout, stderr = job_queue.get_next() - test_results.append(test_result) - done_str = "{}/{} - {}{}{}".format(i + 1, test_count, BOLD[1], test_result.name, BOLD[0]) - if test_result.status == "Passed": - logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time)) - elif test_result.status == "Skipped": - logging.debug("%s skipped" % (done_str)) - else: - print("%s failed, Duration: %s s\n" % (done_str, test_result.time)) - print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') - print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') - if combined_logs_len and os.path.isdir(testdir): - # Print the final `combinedlogslen` lines of the combined logs - print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0])) - print('\n============') - print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) - print('============\n') - combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir] - if BOLD[0]: - combined_logs_args += ['--color'] - combined_logs, _ = subprocess.Popen(combined_logs_args, universal_newlines=True, stdout=subprocess.PIPE).communicate() - print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) - - if failfast: - logging.debug("Early exiting after test failure") - break + i = 0 + while i < test_count: + for test_result, testdir, stdout, stderr in job_queue.get_next(): + test_results.append(test_result) + i += 1 + done_str = "{}/{} - {}{}{}".format(i, test_count, BOLD[1], test_result.name, BOLD[0]) + if test_result.status == "Passed": + logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time)) + elif test_result.status == "Skipped": + logging.debug("%s skipped" % (done_str)) + else: + print("%s failed, Duration: %s s\n" % (done_str, test_result.time)) + print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') + print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') + if combined_logs_len and os.path.isdir(testdir): + # Print the final `combinedlogslen` lines of the combined logs + print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0])) + print('\n============') + print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) + print('============\n') + combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir] + if BOLD[0]: + combined_logs_args += ['--color'] + combined_logs, _ = subprocess.Popen(combined_logs_args, universal_newlines=True, stdout=subprocess.PIPE).communicate() + print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) + + if failfast: + logging.debug("Early exiting after test failure") + break print_results(test_results, max_len_name, (int(time.time() - start_time))) @@ -650,8 +652,9 @@ class TestHandler: dot_count = 0 while True: - # Return first proc that finishes + # Return all procs that have finished, if any. Otherwise sleep until there is one. time.sleep(.5) + ret = [] for job in self.jobs: (name, start_time, proc, testdir, log_out, log_err) = job if proc.poll() is not None: @@ -670,7 +673,9 @@ class TestHandler: clearline = '\r' + (' ' * dot_count) + '\r' print(clearline, end='', flush=True) dot_count = 0 - return TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr + ret.append((TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr)) + if ret: + return ret if self.use_term_control: print('.', end='', flush=True) dot_count += 1 diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index afe4dba7b4..2cb9dc4523 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -390,7 +390,11 @@ class ToolWalletTest(BitcoinTestFramework): bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum3.dump") dump_data["checksum"] = "2" * 10 self.write_dump(dump_data, bad_sum_wallet_dump) - self.assert_raises_tool_error('Error: Dumpfile checksum does not match. Computed {}, expected {}{}'.format(checksum, "2" * 10, "0" * 54), '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') + self.assert_raises_tool_error('Error: Checksum is not the correct size', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + dump_data["checksum"] = "3" * 66 + self.write_dump(dump_data, bad_sum_wallet_dump) + self.assert_raises_tool_error('Error: Checksum is not the correct size', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 0b868dde6c..317121eb68 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -141,7 +141,7 @@ class MultiWalletTest(BitcoinTestFramework): # should raise rpc error if wallet path can't be created err_code = -4 if self.options.descriptors else -1 - assert_raises_rpc_error(err_code, "boost::filesystem::create_directory:", self.nodes[0].createwallet, "w8/bad") + assert_raises_rpc_error(err_code, "boost::filesystem::create_director", self.nodes[0].createwallet, "w8/bad") # check that all requested wallets were created self.stop_node(0) diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 6a8a15d05c..9449b393f1 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -17,6 +17,11 @@ if test -z "$1"; then exit 1 fi +if ! sed --help 2>&1 | grep -q 'GNU'; then + echo "Error: the installed sed package is not compatible. Please make sure you have GNU sed installed in your system."; + exit 1; +fi + RET=0 PREV_BRANCH=$(git name-rev --name-only HEAD) PREV_HEAD=$(git rev-parse HEAD) diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 661bc35175..7d608eed6a 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -43,6 +43,7 @@ KNOWN_VIOLATIONS=( "src/test/dbwrapper_tests.cpp:.*snprintf" "src/test/fuzz/locale.cpp" "src/test/fuzz/string.cpp" + "src/test/util_tests.cpp" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 6ff2008670..393278bd6a 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -10,45 +10,58 @@ signed-integer-overflow:policy/feerate.cpp # -fsanitize=integer suppressions # =============================== +# Dependencies +# ------------ +# Suppressions in dependencies that are developed outside this repository. +unsigned-integer-overflow:*/include/c++/ +unsigned-integer-overflow:bench/bench.h +# unsigned-integer-overflow in FuzzedDataProvider's ConsumeIntegralInRange +unsigned-integer-overflow:FuzzedDataProvider.h +unsigned-integer-overflow:leveldb/ +unsigned-integer-overflow:minisketch/ +unsigned-integer-overflow:test/fuzz/crypto_diff_fuzz_chacha20.cpp +implicit-integer-sign-change:*/include/boost/ +implicit-integer-sign-change:*/include/c++/ +implicit-integer-sign-change:*/new_allocator.h +implicit-integer-sign-change:crc32c/ +# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange +implicit-integer-sign-change:FuzzedDataProvider.h +implicit-integer-sign-change:minisketch/ +implicit-signed-integer-truncation:leveldb/ +implicit-unsigned-integer-truncation:*/include/c++/ +implicit-unsigned-integer-truncation:leveldb/ +implicit-unsigned-integer-truncation:test/fuzz/crypto_diff_fuzz_chacha20.cpp +# std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 +implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage +shift-base:*/include/c++/ +shift-base:leveldb/ +shift-base:minisketch/ +shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp # Unsigned integer overflow occurs when the result of an unsigned integer # computation cannot be represented in its type. Unlike signed integer overflow, # this is not undefined behavior, but it is often unintentional. The list below # contains files in which we expect unsigned integer overflows to occur. The # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. -unsigned-integer-overflow:*/include/c++/ unsigned-integer-overflow:addrman.cpp unsigned-integer-overflow:arith_uint256.h -unsigned-integer-overflow:basic_string.h -unsigned-integer-overflow:bench/bench.h unsigned-integer-overflow:bitcoin-tx.cpp unsigned-integer-overflow:common/bloom.cpp unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h -unsigned-integer-overflow:coded_stream.h unsigned-integer-overflow:coins.cpp unsigned-integer-overflow:compressor.cpp unsigned-integer-overflow:core_write.cpp unsigned-integer-overflow:crypto/ -# unsigned-integer-overflow in FuzzedDataProvider's ConsumeIntegralInRange -unsigned-integer-overflow:FuzzedDataProvider.h unsigned-integer-overflow:hash.cpp -unsigned-integer-overflow:leveldb/ -unsigned-integer-overflow:minisketch/ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h unsigned-integer-overflow:pubkey.h unsigned-integer-overflow:script/interpreter.cpp -unsigned-integer-overflow:stl_bvector.h -unsigned-integer-overflow:test/fuzz/crypto_diff_fuzz_chacha20.cpp unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp -implicit-integer-sign-change:*/include/boost/ -implicit-integer-sign-change:*/include/c++/ -implicit-integer-sign-change:*/new_allocator.h implicit-integer-sign-change:addrman.h -implicit-integer-sign-change:arith_uint256.cpp implicit-integer-sign-change:bech32.cpp implicit-integer-sign-change:common/bloom.cpp implicit-integer-sign-change:chain.cpp @@ -56,55 +69,29 @@ implicit-integer-sign-change:chain.h implicit-integer-sign-change:coins.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h -implicit-integer-sign-change:crc32c/ implicit-integer-sign-change:crypto/ -# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange -implicit-integer-sign-change:FuzzedDataProvider.h implicit-integer-sign-change:key.cpp -implicit-integer-sign-change:minisketch/ implicit-integer-sign-change:noui.cpp implicit-integer-sign-change:policy/fees.cpp implicit-integer-sign-change:prevector.h implicit-integer-sign-change:script/bitcoinconsensus.cpp implicit-integer-sign-change:script/interpreter.cpp implicit-integer-sign-change:serialize.h -implicit-integer-sign-change:test/arith_uint256_tests.cpp -implicit-integer-sign-change:test/coins_tests.cpp -implicit-integer-sign-change:test/pow_tests.cpp -implicit-integer-sign-change:test/prevector_tests.cpp -implicit-integer-sign-change:test/sighash_tests.cpp -implicit-integer-sign-change:test/skiplist_tests.cpp implicit-integer-sign-change:test/streams_tests.cpp -implicit-integer-sign-change:test/transaction_tests.cpp implicit-integer-sign-change:txmempool.cpp implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h -implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp implicit-signed-integer-truncation:addrman.cpp implicit-signed-integer-truncation:addrman.h implicit-signed-integer-truncation:chain.h implicit-signed-integer-truncation:crypto/ -implicit-signed-integer-truncation:leveldb/ implicit-signed-integer-truncation:node/miner.cpp implicit-signed-integer-truncation:net.cpp -implicit-signed-integer-truncation:net_processing.cpp implicit-signed-integer-truncation:streams.h -implicit-signed-integer-truncation:test/arith_uint256_tests.cpp -implicit-signed-integer-truncation:test/skiplist_tests.cpp implicit-signed-integer-truncation:torcontrol.cpp -implicit-unsigned-integer-truncation:*/include/c++/ implicit-unsigned-integer-truncation:crypto/ -implicit-unsigned-integer-truncation:leveldb/ -implicit-unsigned-integer-truncation:test/fuzz/crypto_diff_fuzz_chacha20.cpp -# std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 -implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage -shift-base:*/include/c++/ shift-base:arith_uint256.cpp shift-base:crypto/ shift-base:hash.cpp -shift-base:leveldb/ -shift-base:minisketch/ -shift-base:net_processing.cpp shift-base:streams.h -shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp shift-base:util/bip32.cpp diff --git a/test/util/data/bitcoin-util-test.json b/test/util/data/bitcoin-util-test.json index cca5732aa1..c9c64274c6 100644 --- a/test/util/data/bitcoin-util-test.json +++ b/test/util/data/bitcoin-util-test.json @@ -418,6 +418,29 @@ }, { "exec": "./bitcoin-tx", "args": + ["-create", + "in=00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff:0", + "set=privatekeys:[\"KwDiBf89QgGbjEhKnhXJuH7LrciVrZi3qYjgd9M7rFU73sVHnoWn\"]", + "set=prevtxs:[{\"txid\":\"00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff\",\"vout\":0,\"amount\":\"0\", \"scriptPubKey\":\"0014751e76e8199196d454941c45d1b3a323f1433bd6\"}]", + "sign=ALL", + "outaddr=0.001:193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7"], + "output_cmp": "txcreatesignsegwit1.hex", + "description": "Creates a new transaction with a single witness input and a single output, and then signs the transaction" + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "in=00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff:0", + "set=privatekeys:[\"KwDiBf89QgGbjEhKnhXJuH7LrciVrZi3qYjgd9M7rFU73sVHnoWn\"]", + "set=prevtxs:[{\"txid\":\"00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff\",\"vout\":0,\"scriptPubKey\":\"0014751e76e8199196d454941c45d1b3a323f1433bd6\"}]", + "sign=ALL", + "outaddr=0.001:193P6LtvS4nCnkDvM9uXn1gsSRqh4aDAz7"], + "return_code": 1, + "error_txt": "Missing amount for CTxOut with scriptPubKey=0014751e76e8199196d454941c45d1b3a323f1433bd6", + "description": "Tests the check for missing input amount for witness transactions" + }, + { "exec": "./bitcoin-tx", + "args": ["-create", "outpubkey=0:02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397", "nversion=1"], "output_cmp": "txcreateoutpubkey1.hex", "description": "Creates a new transaction with a single pay-to-pubkey output" diff --git a/test/util/data/txcreatesignsegwit1.hex b/test/util/data/txcreatesignsegwit1.hex new file mode 100644 index 0000000000..45dd1f1dbf --- /dev/null +++ b/test/util/data/txcreatesignsegwit1.hex @@ -0,0 +1 @@ +02000000000101ffeeddccbbaa99887766554433221100ffeeddccbbaa998877665544332211000000000000ffffffff01a0860100000000001976a9145834479edbbe0539b31ffd3a8f8ebadc2165ed0188ac0247304402202e8d8677912f73909ffbdb3ee87d10cce41d398ee206e534fa18330b566ece34022004f944f018a03c9f5b4cf0e9b0ae4f14049b55e7b6810a6ac26cd67cb4dcb31f01210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179800000000 |
