diff options
162 files changed, 2546 insertions, 1424 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 446d3e35a9..33bf43d4b1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -5,7 +5,7 @@ container: # https://cirrus-ci.org/faq/#are-there-any-limits # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel cpu: 2 - memory: 6G # https://cirrus-ci.org/guide/linux/#linux-containers + memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers env: PACKAGE_MANAGER_INSTALL : "apt-get update && apt-get install -y" MAKEJOBS: "-j4" @@ -66,3 +66,11 @@ task: image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" + +task: + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:focal + env: + FILE_ENV: "./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md index 8be78a1f6e..d32e22d360 100644 --- a/.github/ISSUE_TEMPLATE/good_first_issue.md +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -2,11 +2,13 @@ name: Good first issue about: '(Regular devs only): Suggest a new good first issue' title: '' -labels: good first issue +labels: '' assignees: '' --- +<!-- Needs the label "good first issue" assigned manually before or after opening --> + <!-- A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution --> <!-- Motivate the issue and explain the solution briefly --> diff --git a/.travis.yml b/.travis.yml index f1cee7133f..8f8fba3108 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,11 +110,6 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" - - - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" diff --git a/build-aux/m4/ax_boost_thread.m4 b/build-aux/m4/ax_boost_thread.m4 index e9dea43535..75e80e6e75 100644 --- a/build-aux/m4/ax_boost_thread.m4 +++ b/build-aux/m4/ax_boost_thread.m4 @@ -30,7 +30,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 32 +#serial 33 AC_DEFUN([AX_BOOST_THREAD], [ @@ -67,13 +67,24 @@ AC_DEFUN([AX_BOOST_THREAD], [AC_LANG_PUSH([C++]) CXXFLAGS_SAVE=$CXXFLAGS - if test "x$host_os" = "xsolaris" ; then - CXXFLAGS="-pthreads $CXXFLAGS" - elif test "x$host_os" = "xmingw32" ; then - CXXFLAGS="-mthreads $CXXFLAGS" - else - CXXFLAGS="-pthread $CXXFLAGS" - fi + case "x$host_os" in + xsolaris ) + CXXFLAGS="-pthreads $CXXFLAGS" + break; + ;; + xmingw32 ) + CXXFLAGS="-mthreads $CXXFLAGS" + break; + ;; + *android* ) + break; + ;; + * ) + CXXFLAGS="-pthread $CXXFLAGS" + break; + ;; + esac + AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [[@%:@include <boost/thread/thread.hpp>]], @@ -84,13 +95,23 @@ AC_DEFUN([AX_BOOST_THREAD], AC_LANG_POP([C++]) ]) if test "x$ax_cv_boost_thread" = "xyes"; then - if test "x$host_os" = "xsolaris" ; then - BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS" - elif test "x$host_os" = "xmingw32" ; then - BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS" - else - BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS" - fi + case "x$host_os" in + xsolaris ) + BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS" + break; + ;; + xmingw32 ) + BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS" + break; + ;; + *android* ) + break; + ;; + * ) + BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS" + break; + ;; + esac AC_SUBST(BOOST_CPPFLAGS) @@ -148,6 +169,9 @@ AC_DEFUN([AX_BOOST_THREAD], xmingw32 ) break; ;; + *android* ) + break; + ;; * ) BOOST_THREAD_LIB="$BOOST_THREAD_LIB -lpthread" break; diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index e171238cbc..6c7665830b 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -72,18 +72,32 @@ AC_DEFUN([BITCOIN_QT_INIT],[ AC_ARG_WITH([qtdbus], [AS_HELP_STRING([--with-qtdbus], - [enable DBus support (default is yes if qt is enabled and QtDBus is found)])], + [enable DBus support (default is yes if qt is enabled and QtDBus is found, except on Android)])], [use_dbus=$withval], [use_dbus=auto]) + dnl Android doesn't support D-Bus and certainly doesn't use it for notifications + case $host in + *android*) + if test "x$use_dbus" != xyes; then + use_dbus=no + fi + ;; + esac + AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path) ]) dnl Find Qt libraries and includes. +dnl +dnl BITCOIN_QT_CONFIGURE([MINIMUM-VERSION]) +dnl dnl Outputs: See _BITCOIN_QT_FIND_LIBS dnl Outputs: Sets variables for all qt-related tools. dnl Outputs: bitcoin_enable_qt, bitcoin_enable_qt_dbus, bitcoin_enable_qt_test AC_DEFUN([BITCOIN_QT_CONFIGURE],[ + qt_version=">= $1" + qt_lib_prefix="Qt5" BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS]) dnl This is ugly and complicated. Yuck. Works as follows: @@ -221,7 +235,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ bitcoin_enable_qt=no ]) if test x$bitcoin_enable_qt = xyes; then - AC_MSG_RESULT([$bitcoin_enable_qt ($QT_LIB_PREFIX)]) + AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)]) else AC_MSG_RESULT([$bitcoin_enable_qt]) fi @@ -295,25 +309,19 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/platforms/android"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" fi - m4_ifdef([PKG_CHECK_MODULES],[ - if test x$bitcoin_cv_qt58 = xno; then - PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) - else - PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) - fi - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then - PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) - fi - ]) + PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) + if test "x$TARGET_OS" = xlinux; then + PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) + fi fi ]) @@ -321,23 +329,29 @@ dnl Internal. Find Qt libraries using pkg-config. dnl Outputs: All necessary QT_* variables are set. dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ - m4_ifdef([PKG_CHECK_MODULES],[ - QT_LIB_PREFIX=Qt5 - qt5_modules="Qt5Core Qt5Gui Qt5Network Qt5Widgets" - BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT5], [$qt5_modules], [QT_INCLUDES="$QT5_CFLAGS"; QT_LIBS="$QT5_LIBS" have_qt=yes],[have_qt=no]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_CORE], [${qt_lib_prefix}Core $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Core $qt_version not found])]) + ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_GUI], [${qt_lib_prefix}Gui $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Gui $qt_version not found])]) + ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_WIDGETS], [${qt_lib_prefix}Widgets $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Widgets $qt_version not found])]) + ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_NETWORK], [${qt_lib_prefix}Network $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Network $qt_version not found])]) + ]) + QT_INCLUDES="$QT_CORE_CFLAGS $QT_GUI_CFLAGS $QT_WIDGETS_CFLAGS $QT_NETWORK_CFLAGS" + QT_LIBS="$QT_CORE_LIBS $QT_GUI_LIBS $QT_WIDGETS_LIBS $QT_NETWORK_LIBS" - if test "x$have_qt" != xyes; then - have_qt=no - BITCOIN_QT_FAIL([Qt dependencies not found]) - fi - ]) - BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_TEST], [${QT_LIB_PREFIX}Test], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) - if test "x$use_dbus" != xno; then - PKG_CHECK_MODULES([QT_DBUS], [${QT_LIB_PREFIX}DBus], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) - fi - ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) + if test "x$use_dbus" != xno; then + PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) + fi ]) - true; dnl ]) diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 4fd516fff5..a080fd2aa4 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -110,7 +110,7 @@ <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> diff --git a/configure.ac b/configure.ac index 2acd702600..acd4e0cf6c 100644 --- a/configure.ac +++ b/configure.ac @@ -377,6 +377,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=shadow-field],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=shadow-field"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=range-loop-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=range-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) @@ -405,6 +406,10 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-branches],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-cond],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) @@ -599,7 +604,7 @@ case $host in AC_MSG_ERROR("windres not found") fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601" + CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -1156,7 +1161,7 @@ else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE + BITCOIN_QT_CONFIGURE([5.5.1]) fi if test x$enable_wallet != xno; then diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index b679438c6f..06cf974f75 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://download.oracle.com/berkeley-db $(package)_file_name=db-$($(package)_version).NC.tar.gz $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef $(package)_build_subdir=build_unix +$(package)_patches=clang_cxx_11.patch define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking @@ -14,8 +15,7 @@ $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef define $(package)_preprocess_cmds - sed -i.old 's/__atomic_compare_exchange/__atomic_compare_exchange_db/' dbinc/atomic.h && \ - sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c && \ + patch -p1 < $($(package)_patch_dir)/clang_cxx_11.patch && \ cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub dist endef diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 4f6b543aff..d8bce108b1 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -3,6 +3,7 @@ $(package)_version=1_70_0 $(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/ $(package)_file_name=boost_$($(package)_version).tar.bz2 $(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 +$(package)_patches=unused_var_in_process.patch define $(package)_set_vars $(package)_config_opts_release=variant=release @@ -31,9 +32,8 @@ $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC endef -# Fix unused variable in boost_process, can be removed after upgrading to 1.72 define $(package)_preprocess_cmds - sed -i.old "s/int ret_sig = 0;//" boost/process/detail/posix/wait_group.hpp && \ + patch -p1 < $($(package)_patch_dir)/unused_var_in_process.patch && \ echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam endef diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 128599ba77..0d5f94f380 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -4,23 +4,23 @@ $(package)_download_path=https://www.freedesktop.org/software/fontconfig/release $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3 $(package)_dependencies=freetype expat +$(package)_patches=remove_char_width_usage.patch gperf_header_regen.patch define $(package)_set_vars $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv $(package)_config_opts += --disable-dependency-tracking --enable-option-checking endef +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/remove_char_width_usage.patch && \ + patch -p1 < $($(package)_patch_dir)/gperf_header_regen.patch +endef + define $(package)_config_cmds $($(package)_autoconf) endef -# 2.12.1 uses CHAR_WIDTH which is reserved and clashes with some glibc versions, but newer versions of fontconfig -# have broken makefiles which needlessly attempt to re-generate headers with gperf. -# Instead, change all uses of CHAR_WIDTH, and disable the rule that forces header re-generation. -# This can be removed once the upstream build is fixed. define $(package)_build_cmds - sed -i 's/CHAR_WIDTH/CHARWIDTH/g' fontconfig/fontconfig.h src/fcobjshash.gperf src/fcobjs.h src/fcobjshash.h && \ - sed -i 's/fcobjshash.h: fcobjshash.gperf/fcobjshash.h:/' src/Makefile && \ $(MAKE) endef diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index fdbe22cda6..49a584e462 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -3,6 +3,7 @@ $(package)_version=2.0.20180203 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7 +$(package)_patches=dont_use_wingen.patch define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" @@ -14,7 +15,7 @@ endef define $(package)_preprocess_cmds mkdir dll && \ sed -e 's|MINIUPNPC_VERSION_STRING \"version\"|MINIUPNPC_VERSION_STRING \"$($(package)_version)\"|' -e 's|OS/version|$(host)|' miniupnpcstrings.h.in > miniupnpcstrings.h && \ - sed -i.old "s|miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings|miniupnpcstrings.h: miniupnpcstrings.h.in|" Makefile.mingw + patch -p1 < $($(package)_patch_dir)/dont_use_wingen.patch endef define $(package)_build_cmds diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 5022ed980f..d56b636695 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -4,6 +4,8 @@ $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875 $(package)_build_subdir=cctools +$(package)_patches=ld64_disable_threading.patch + ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) $(package)_clang_version=8.0.0 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) @@ -78,7 +80,7 @@ endef define $(package)_preprocess_cmds CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \ CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \ - sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h + patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch endef define $(package)_config_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 500881e442..f560099b6a 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -8,7 +8,10 @@ $(package)_dependencies=zlib $(package)_linux_dependencies=freetype fontconfig libxcb $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch +$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch +$(package)_patches+= fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch +$(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch +$(package)_patches+= freetype_back_compat.patch drop_lrelease_dependency.patch # Update OSX_QT_TRANSLATIONS when this is updated $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) @@ -190,11 +193,10 @@ define $(package)_extract_cmds endef define $(package)_preprocess_cmds - sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \ + patch -p1 -i $($(package)_patch_dir)/freetype_back_compat.patch && \ sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ - sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ - sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ - sed -i.old -e 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' -e 's|/bin/pwd|pwd|' qtbase/configure && \ + patch -p1 -i $($(package)_patch_dir)/drop_lrelease_dependency.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch &&\ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 6f35ede248..c93aa1a74d 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -3,7 +3,7 @@ $(package)_version=4.3.1 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb -$(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch +$(package)_patches=remove_libstd_link.patch define $(package)_set_vars $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf @@ -16,9 +16,8 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/0001-fix-build-with-older-mingw64.patch && \ - patch -p1 < $($(package)_patch_dir)/0002-disable-pthread_set_name_np.patch && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config + patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds @@ -34,6 +33,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \ rm -rf bin share lib/*.la endef diff --git a/depends/patches/bdb/clang_cxx_11.patch b/depends/patches/bdb/clang_cxx_11.patch new file mode 100644 index 0000000000..58f7ddc7d5 --- /dev/null +++ b/depends/patches/bdb/clang_cxx_11.patch @@ -0,0 +1,147 @@ +commit 3311d68f11d1697565401eee6efc85c34f022ea7 +Author: fanquake <fanquake@gmail.com> +Date: Mon Aug 17 20:03:56 2020 +0800 + + Fix C++11 compatibility + +diff --git a/dbinc/atomic.h b/dbinc/atomic.h +index 0034dcc..7c11d4a 100644 +--- a/dbinc/atomic.h ++++ b/dbinc/atomic.h +@@ -70,7 +70,7 @@ typedef struct { + * These have no memory barriers; the caller must include them when necessary. + */ + #define atomic_read(p) ((p)->value) +-#define atomic_init(p, val) ((p)->value = (val)) ++#define atomic_init_db(p, val) ((p)->value = (val)) + + #ifdef HAVE_ATOMIC_SUPPORT + +@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; + #define atomic_inc(env, p) __atomic_inc(p) + #define atomic_dec(env, p) __atomic_dec(p) + #define atomic_compare_exchange(env, p, o, n) \ +- __atomic_compare_exchange((p), (o), (n)) ++ __atomic_compare_exchange_db((p), (o), (n)) + static inline int __atomic_inc(db_atomic_t *p) + { + int temp; +@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) + * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html + * which configure could be changed to use. + */ +-static inline int __atomic_compare_exchange( ++static inline int __atomic_compare_exchange_db( + db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) + { + atomic_value_t was; +@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( + #define atomic_dec(env, p) (--(p)->value) + #define atomic_compare_exchange(env, p, oldval, newval) \ + (DB_ASSERT(env, atomic_read(p) == (oldval)), \ +- atomic_init(p, (newval)), 1) ++ atomic_init_db(p, (newval)), 1) + #else + #define atomic_inc(env, p) __atomic_inc(env, p) + #define atomic_dec(env, p) __atomic_dec(env, p) +diff --git a/mp/mp_fget.c b/mp/mp_fget.c +index 5fdee5a..0b75f57 100644 +--- a/mp/mp_fget.c ++++ b/mp/mp_fget.c +@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ + + /* Initialize enough so we can call __memp_bhfree. */ + alloc_bhp->flags = 0; +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + #ifdef DIAGNOSTIC + if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { + __db_errx(env, +@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ + MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, + PROT_READ); + +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + MUTEX_LOCK(env, alloc_bhp->mtx_buf); + alloc_bhp->priority = bhp->priority; + alloc_bhp->pgno = bhp->pgno; +diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c +index 34467d2..f05aa0c 100644 +--- a/mp/mp_mvcc.c ++++ b/mp/mp_mvcc.c +@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) + #else + memcpy(frozen_bhp, bhp, SSZA(BH, buf)); + #endif +- atomic_init(&frozen_bhp->ref, 0); ++ atomic_init_db(&frozen_bhp->ref, 0); + if (mutex != MUTEX_INVALID) + frozen_bhp->mtx_buf = mutex; + else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, +@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) + #endif + alloc_bhp->mtx_buf = mutex; + MUTEX_LOCK(env, alloc_bhp->mtx_buf); +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + F_CLR(alloc_bhp, BH_FROZEN); + } + +diff --git a/mp/mp_region.c b/mp/mp_region.c +index e6cece9..ddbe906 100644 +--- a/mp/mp_region.c ++++ b/mp/mp_region.c +@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) + return (ret); + SH_TAILQ_INIT(&htab[i].hash_bucket); +- atomic_init(&htab[i].hash_page_dirty, 0); ++ atomic_init_db(&htab[i].hash_page_dirty, 0); + } + + /* +@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : + mtx_base + i; + SH_TAILQ_INIT(&hp->hash_bucket); +- atomic_init(&hp->hash_page_dirty, 0); ++ atomic_init_db(&hp->hash_page_dirty, 0); + #ifdef HAVE_STATISTICS + hp->hash_io_wait = 0; + hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; +diff --git a/mutex/mut_method.c b/mutex/mut_method.c +index 2588763..5c6d516 100644 +--- a/mutex/mut_method.c ++++ b/mutex/mut_method.c +@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) + MUTEX_LOCK(env, mtx); + ret = atomic_read(v) == oldval; + if (ret) +- atomic_init(v, newval); ++ atomic_init_db(v, newval); + MUTEX_UNLOCK(env, mtx); + + return (ret); +diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c +index f3922e0..e40fcdf 100644 +--- a/mutex/mut_tas.c ++++ b/mutex/mut_tas.c +@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) + + #ifdef HAVE_SHARED_LATCHES + if (F_ISSET(mutexp, DB_MUTEX_SHARED)) +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + else + #endif + if (MUTEX_INIT(&mutexp->tas)) { +@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) + F_CLR(mutexp, DB_MUTEX_LOCKED); + /* Flush flag update before zeroing count */ + MEMBAR_EXIT(); +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + } else { + DB_ASSERT(env, sharecount > 0); + MEMBAR_EXIT(); diff --git a/depends/patches/boost/unused_var_in_process.patch b/depends/patches/boost/unused_var_in_process.patch new file mode 100644 index 0000000000..722f7bb5ea --- /dev/null +++ b/depends/patches/boost/unused_var_in_process.patch @@ -0,0 +1,22 @@ +commit dbd95cdaefdea95307d004f019a1c394cf9389f0 +Author: fanquake <fanquake@gmail.com> +Date: Mon Aug 17 20:15:17 2020 +0800 + + Remove unused variable in Boost Process + + This causes issues with our linters / CI. + + Can be removed once depends Boost is 1.71.0 or later. + +diff --git a/boost/process/detail/posix/wait_group.hpp b/boost/process/detail/posix/wait_group.hpp +index 9dc249803..2502d9772 100644 +--- a/boost/process/detail/posix/wait_group.hpp ++++ b/boost/process/detail/posix/wait_group.hpp +@@ -137,7 +137,6 @@ inline bool wait_until( + + do + { +- int ret_sig = 0; + int status; + if ((::waitpid(timeout_pid, &status, WNOHANG) != 0) + && (WIFEXITED(status) || WIFSIGNALED(status))) diff --git a/depends/patches/fontconfig/gperf_header_regen.patch b/depends/patches/fontconfig/gperf_header_regen.patch new file mode 100644 index 0000000000..7401b83d84 --- /dev/null +++ b/depends/patches/fontconfig/gperf_header_regen.patch @@ -0,0 +1,24 @@ +commit 7b6eb33ecd88768b28c67ce5d2d68a7eed5936b6 +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 25 14:34:53 2020 +0800 + + Remove rule that causes inadvertant header regeneration + + Otherwise the makefile will needlessly attempt to re-generate the + headers with gperf. This can be dropped once the upstream build is fixed. + + See #10851. + +diff --git a/src/Makefile.in b/src/Makefile.in +index f4626ad..4ae1b00 100644 +--- a/src/Makefile.in ++++ b/src/Makefile.in +@@ -903,7 +903,7 @@ fcobjshash.gperf: fcobjshash.gperf.h fcobjs.h + ' - > $@.tmp && \ + mv -f $@.tmp $@ || ( $(RM) $@.tmp && false ) + +-fcobjshash.h: fcobjshash.gperf ++fcobjshash.h: + $(AM_V_GEN) $(GPERF) -m 100 $< > $@.tmp && \ + mv -f $@.tmp $@ || ( $(RM) $@.tmp && false ) + diff --git a/depends/patches/fontconfig/remove_char_width_usage.patch b/depends/patches/fontconfig/remove_char_width_usage.patch new file mode 100644 index 0000000000..9f69081890 --- /dev/null +++ b/depends/patches/fontconfig/remove_char_width_usage.patch @@ -0,0 +1,62 @@ +commit 28165a9b078583dc8e9e5c344510e37582284cef +Author: fanquake <fanquake@gmail.com> +Date: Mon Aug 17 20:35:42 2020 +0800 + + Remove usage of CHAR_WIDTH + + CHAR_WIDTH which is reserved and clashes with glibc 2.25+ + + See #10851. + +diff --git a/fontconfig/fontconfig.h b/fontconfig/fontconfig.h +index 5c72b22..843c532 100644 +--- a/fontconfig/fontconfig.h ++++ b/fontconfig/fontconfig.h +@@ -128,7 +128,7 @@ typedef int FcBool; + #define FC_USER_CACHE_FILE ".fonts.cache-" FC_CACHE_VERSION + + /* Adjust outline rasterizer */ +-#define FC_CHAR_WIDTH "charwidth" /* Int */ ++#define FC_CHARWIDTH "charwidth" /* Int */ + #define FC_CHAR_HEIGHT "charheight"/* Int */ + #define FC_MATRIX "matrix" /* FcMatrix */ + +diff --git a/src/fcobjs.h b/src/fcobjs.h +index 1fc4f65..d27864b 100644 +--- a/src/fcobjs.h ++++ b/src/fcobjs.h +@@ -51,7 +51,7 @@ FC_OBJECT (DPI, FcTypeDouble, NULL) + FC_OBJECT (RGBA, FcTypeInteger, NULL) + FC_OBJECT (SCALE, FcTypeDouble, NULL) + FC_OBJECT (MINSPACE, FcTypeBool, NULL) +-FC_OBJECT (CHAR_WIDTH, FcTypeInteger, NULL) ++FC_OBJECT (CHARWIDTH, FcTypeInteger, NULL) + FC_OBJECT (CHAR_HEIGHT, FcTypeInteger, NULL) + FC_OBJECT (MATRIX, FcTypeMatrix, NULL) + FC_OBJECT (CHARSET, FcTypeCharSet, FcCompareCharSet) +diff --git a/src/fcobjshash.gperf b/src/fcobjshash.gperf +index 80a0237..eb4ad84 100644 +--- a/src/fcobjshash.gperf ++++ b/src/fcobjshash.gperf +@@ -44,7 +44,7 @@ int id; + "rgba",FC_RGBA_OBJECT + "scale",FC_SCALE_OBJECT + "minspace",FC_MINSPACE_OBJECT +-"charwidth",FC_CHAR_WIDTH_OBJECT ++"charwidth",FC_CHARWIDTH_OBJECT + "charheight",FC_CHAR_HEIGHT_OBJECT + "matrix",FC_MATRIX_OBJECT + "charset",FC_CHARSET_OBJECT +diff --git a/src/fcobjshash.h b/src/fcobjshash.h +index 5a4d1ea..4e66bb0 100644 +--- a/src/fcobjshash.h ++++ b/src/fcobjshash.h +@@ -284,7 +284,7 @@ FcObjectTypeLookup (register const char *str, register unsigned int len) + {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str43,FC_CHARSET_OBJECT}, + {-1}, + #line 47 "fcobjshash.gperf" +- {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHAR_WIDTH_OBJECT}, ++ {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHARWIDTH_OBJECT}, + #line 48 "fcobjshash.gperf" + {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str46,FC_CHAR_HEIGHT_OBJECT}, + #line 55 "fcobjshash.gperf" diff --git a/depends/patches/miniupnpc/dont_use_wingen.patch b/depends/patches/miniupnpc/dont_use_wingen.patch new file mode 100644 index 0000000000..a1cc9b50d1 --- /dev/null +++ b/depends/patches/miniupnpc/dont_use_wingen.patch @@ -0,0 +1,26 @@ +commit e8077044df239bcf0d9e9980b0e1afb9f1f5c446 +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 18 20:50:19 2020 +0800 + + Don't use wingenminiupnpcstrings when generating miniupnpcstrings.h + + The wingenminiupnpcstrings tool is used on Windows to generate version + information. This information is irrelevant for us, and trying to use + wingenminiupnpcstrings would cause builds to fail, so just don't use it. + + We should be able to drop this once we are using 2.1 or later. See + upstream commit: 9663c55c61408fdcc39a82987d2243f816b22932. + +diff --git a/Makefile.mingw b/Makefile.mingw +index 574720e..fcc17bb 100644 +--- a/Makefile.mingw ++++ b/Makefile.mingw +@@ -74,7 +74,7 @@ wingenminiupnpcstrings: wingenminiupnpcstrings.o + + wingenminiupnpcstrings.o: wingenminiupnpcstrings.c + +-miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings ++miniupnpcstrings.h: miniupnpcstrings.h.in + wingenminiupnpcstrings $< $@ + + minixml.o: minixml.c minixml.h diff --git a/depends/patches/native_cctools/ld64_disable_threading.patch b/depends/patches/native_cctools/ld64_disable_threading.patch new file mode 100644 index 0000000000..d6c58c102f --- /dev/null +++ b/depends/patches/native_cctools/ld64_disable_threading.patch @@ -0,0 +1,26 @@ +commit 584668415039adeed073decee7e04de28248afd3 +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 18 01:20:24 2020 +0000 + + Disable threading to fix non-determinism + + A bug in the file parser can cause dependencies to be calculated + differently based on which files have already been parsed. This is more + likely to occur on systems with more CPUs. + + Just disable threading for now. There is no noticable slowdown. + + See #9891. + +diff --git a/cctools/ld64/src/ld/InputFiles.h b/cctools/ld64/src/ld/InputFiles.h +index ef9c756..90a70b6 100644 +--- a/cctools/ld64/src/ld/InputFiles.h ++++ b/cctools/ld64/src/ld/InputFiles.h +@@ -25,7 +25,6 @@ + #ifndef __INPUT_FILES_H__ + #define __INPUT_FILES_H__ + +-#define HAVE_PTHREADS 1 + + #include <stdlib.h> + #include <sys/types.h> diff --git a/depends/patches/qt/dont_hardcode_pwd.patch b/depends/patches/qt/dont_hardcode_pwd.patch new file mode 100644 index 0000000000..a74e9cb098 --- /dev/null +++ b/depends/patches/qt/dont_hardcode_pwd.patch @@ -0,0 +1,27 @@ +commit 0e953866fc4672486e29e1ba6d83b4207e7b2f0b +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 18 15:09:06 2020 +0800 + + Don't hardcode pwd path + + Let a man use his builtins if he wants to! Also, removes the unnecessary + assumption that pwd lives under /bin/pwd. + + See #15581. + +diff --git a/qtbase/configure b/qtbase/configure +index 08b49a8d..faea5b55 100755 +--- a/qtbase/configure ++++ b/qtbase/configure +@@ -36,9 +36,9 @@ + relconf=`basename $0` + # the directory of this script is the "source tree" + relpath=`dirname $0` +-relpath=`(cd "$relpath"; /bin/pwd)` ++relpath=`(cd "$relpath"; pwd)` + # the current directory is the "build tree" or "object tree" +-outpath=`/bin/pwd` ++outpath=`pwd` + + WHICH="which" + diff --git a/depends/patches/qt/drop_lrelease_dependency.patch b/depends/patches/qt/drop_lrelease_dependency.patch new file mode 100644 index 0000000000..f6b2c9fc80 --- /dev/null +++ b/depends/patches/qt/drop_lrelease_dependency.patch @@ -0,0 +1,20 @@ +commit 67b3ed7406e1d0762188dbad2c44a06824ba0778 +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 18 15:24:01 2020 +0800 + + Drop dependency on lrelease + + Qts buildsystem insists on using the installed lrelease, but gets + confused about how to find it. Since we manually control the build + order, just drop the dependency. + + See #9469 + +diff --git a/qttranslations/translations/translations.pro b/qttranslations/translations/translations.pro +index 694544c..eff339d 100644 +--- a/qttranslations/translations/translations.pro ++++ b/qttranslations/translations/translations.pro +@@ -109,3 +109,2 @@ updateqm.commands = $$LRELEASE ${QMAKE_FILE_IN} -qm ${QMAKE_FILE_OUT} + silent:updateqm.commands = @echo lrelease ${QMAKE_FILE_IN} && $$updateqm.commands +-updateqm.depends = $$LRELEASE_EXE + updateqm.name = LRELEASE ${QMAKE_FILE_IN} diff --git a/depends/patches/qt/freetype_back_compat.patch b/depends/patches/qt/freetype_back_compat.patch new file mode 100644 index 0000000000..b0f1c98aa6 --- /dev/null +++ b/depends/patches/qt/freetype_back_compat.patch @@ -0,0 +1,28 @@ +commit 14bc77db61bf9d56f9b6c8b84aa02573605c19c6 +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 18 15:15:08 2020 +0800 + + Fix backwards compatibility with older Freetype versions at runtime + + A few years ago, libfreetype introduced FT_Get_Font_Format() as an alias + for FT_Get_X11_Font_Format(), but FT_Get_X11_Font_Format() was kept for abi + backwards-compatibility. + + Qt 5.9 introduced a call to FT_Get_Font_Format(). Replace it with FT_Get_X11_Font_Format() + in order to remain compatibile with older freetype, which is still used by e.g. Ubuntu Trusty. + + See #14348. + +diff --git a/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp b/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp +index 3f543755..8ecc1c8c 100644 +--- a/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp ++++ b/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp +@@ -898,7 +898,7 @@ bool QFontEngineFT::init(FaceId faceId, bool antialias, GlyphFormat format, + } + } + #if defined(FT_FONT_FORMATS_H) +- const char *fmt = FT_Get_Font_Format(face); ++ const char *fmt = FT_Get_X11_Font_Format(face); + if (fmt && qstrncmp(fmt, "CFF", 4) == 0) { + FT_Bool no_stem_darkening = true; + FT_Error err = FT_Property_Get(qt_getFreetype(), "cff", "no-stem-darkening", &no_stem_darkening); diff --git a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch deleted file mode 100644 index b911ac5672..0000000000 --- a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch +++ /dev/null @@ -1,30 +0,0 @@ -From f6866b0f166ad168618aae64c7fbee8775d3eb23 Mon Sep 17 00:00:00 2001 -From: mruddy <6440430+mruddy@users.noreply.github.com> -Date: Sat, 30 Jun 2018 09:44:58 -0400 -Subject: [PATCH] fix build with older mingw64 - ---- - src/windows.hpp | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/src/windows.hpp b/src/windows.hpp -index 6c3839fd..2c32ec79 100644 ---- a/src/windows.hpp -+++ b/src/windows.hpp -@@ -58,6 +58,13 @@ - #include <winsock2.h> - #include <windows.h> - #include <mswsock.h> -+ -+#if defined __MINGW64_VERSION_MAJOR && __MINGW64_VERSION_MAJOR < 4 -+// Workaround for mingw-w64 < v4.0 which did not include ws2ipdef.h in iphlpapi.h. -+// Fixed in mingw-w64 by 9bd8fe9148924840d315b4c915dd099955ea89d1. -+#include <ws2def.h> -+#include <ws2ipdef.h> -+#endif - #include <iphlpapi.h> - - #if !defined __MINGW32__ --- -2.17.1 - diff --git a/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch b/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch deleted file mode 100644 index b1c6f78a70..0000000000 --- a/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch +++ /dev/null @@ -1,35 +0,0 @@ -From c9bbdd6581d07acfe8971e4bcebe278a3676cf03 Mon Sep 17 00:00:00 2001 -From: mruddy <6440430+mruddy@users.noreply.github.com> -Date: Sat, 30 Jun 2018 09:57:18 -0400 -Subject: [PATCH] disable pthread_set_name_np - -pthread_set_name_np adds a Glibc requirement on >= 2.12. ---- - src/thread.cpp | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/src/thread.cpp b/src/thread.cpp -index a1086b0c..9943f354 100644 ---- a/src/thread.cpp -+++ b/src/thread.cpp -@@ -308,7 +308,7 @@ void zmq::thread_t::setThreadName (const char *name_) - */ - if (!name_) - return; -- -+#if 0 - #if defined(ZMQ_HAVE_PTHREAD_SETNAME_1) - int rc = pthread_setname_np (name_); - if (rc) -@@ -324,6 +324,8 @@ void zmq::thread_t::setThreadName (const char *name_) - #elif defined(ZMQ_HAVE_PTHREAD_SET_NAME) - pthread_set_name_np (_descriptor, name_); - #endif -+#endif -+ return; - } - - #endif --- -2.17.1 - diff --git a/depends/patches/zeromq/remove_libstd_link.patch b/depends/patches/zeromq/remove_libstd_link.patch new file mode 100644 index 0000000000..ddf91e6abf --- /dev/null +++ b/depends/patches/zeromq/remove_libstd_link.patch @@ -0,0 +1,25 @@ +commit 47d4cd12a2c051815ddda78adebdb3923b260d8a +Author: fanquake <fanquake@gmail.com> +Date: Tue Aug 18 14:45:40 2020 +0800 + + Remove needless linking against libstdc++ + + This is broken for a number of reasons, including: + - g++ understands "static-libstdc++ -lstdc++" to mean "link against + whatever libstdc++ exists, probably shared", which in itself is buggy. + - another stdlib (libc++ for example) may be in use + + See #11981. + +diff --git a/src/libzmq.pc.in b/src/libzmq.pc.in +index 233bc3a..3c2bf0d 100644 +--- a/src/libzmq.pc.in ++++ b/src/libzmq.pc.in +@@ -7,6 +7,6 @@ Name: libzmq + Description: 0MQ c++ library + Version: @VERSION@ + Libs: -L${libdir} -lzmq +-Libs.private: -lstdc++ @pkg_config_libs_private@ ++Libs.private: @pkg_config_libs_private@ + Requires.private: @pkg_config_names_private@ + Cflags: -I${includedir} @pkg_config_defines@ diff --git a/doc/release-notes-14582.md b/doc/release-notes-14582.md new file mode 100644 index 0000000000..28b0abecd7 --- /dev/null +++ b/doc/release-notes-14582.md @@ -0,0 +1,14 @@ +Configuration +------------- + +A new configuration flag `-maxapsfee` has been added, which sets the max allowed +avoid partial spends (APS) fee. It defaults to 0 (i.e. fee is the same with +and without APS). Setting it to -1 will disable APS, unless `-avoidpartialspends` +is set. (#14582) + +Wallet +------ + +The wallet will now avoid partial spends (APS) by default, if this does not result +in a difference in fees compared to the non-APS variant. The allowed fee threshold +can be adjusted using the new `-maxapsfee` configuration option. (#14582) diff --git a/doc/release-notes-19731.md b/doc/release-notes-19731.md new file mode 100644 index 0000000000..abe38e06af --- /dev/null +++ b/doc/release-notes-19731.md @@ -0,0 +1,6 @@ +Updated RPCs +------------ + +- The `getpeerinfo` RPC now has additional `last_block` and `last_transaction` + fields that return the UNIX epoch time of the last block and the last valid + transaction received from each peer. (#19731) diff --git a/doc/release-notes.md b/doc/release-notes.md index 23983dcd7b..a8bd68370d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -115,6 +115,10 @@ Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section New RPCs -------- +- The `getindexinfo` RPC returns the actively running indices of the node, + including their current sync status and height. It also accepts an `index_name` + to specify returning only the status of that index. (#19550) + Build System ------------ @@ -135,6 +139,10 @@ Updated settings in future releases. Refer to the help of the affected settings `-whitebind` and `-whitelist` for more details. (#19191) +- Netmasks that contain 1-bits after 0-bits (the 1-bits are not contiguous on + the left side, e.g. 255.0.255.255) are no longer accepted. They are invalid + according to RFC 4632. + Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. Tools and Utilities diff --git a/src/Makefile.am b/src/Makefile.am index f52e9e5811..175501d4a6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -140,6 +140,7 @@ BITCOIN_CORE_H = \ httpserver.h \ index/base.h \ index/blockfilterindex.h \ + index/disktxpos.h \ index/txindex.h \ indirectmap.h \ init.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0068c94070..a0c9c30f36 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -70,6 +70,7 @@ FUZZ_TARGETS = \ test/fuzz/message \ test/fuzz/messageheader_deserialize \ test/fuzz/multiplication_overflow \ + test/fuzz/net \ test/fuzz/net_permissions \ test/fuzz/netaddr_deserialize \ test/fuzz/netaddress \ @@ -722,6 +723,12 @@ test_fuzz_multiplication_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_multiplication_overflow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_multiplication_overflow_SOURCES = test/fuzz/multiplication_overflow.cpp +test_fuzz_net_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_net_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_net_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_net_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_net_SOURCES = test/fuzz/net.cpp + test_fuzz_net_permissions_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_net_permissions_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_net_permissions_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/base58.cpp b/src/base58.cpp index 3a28088079..0dc6044145 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -84,21 +84,21 @@ NODISCARD static bool DecodeBase58(const char* psz, std::vector<unsigned char>& return true; } -std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) +std::string EncodeBase58(Span<const unsigned char> input) { // Skip & count leading zeroes. int zeroes = 0; int length = 0; - while (pbegin != pend && *pbegin == 0) { - pbegin++; + while (input.size() > 0 && input[0] == 0) { + input = input.subspan(1); zeroes++; } // Allocate enough space in big-endian base58 representation. - int size = (pend - pbegin) * 138 / 100 + 1; // log(256) / log(58), rounded up. + int size = input.size() * 138 / 100 + 1; // log(256) / log(58), rounded up. std::vector<unsigned char> b58(size); // Process the bytes. - while (pbegin != pend) { - int carry = *pbegin; + while (input.size() > 0) { + int carry = input[0]; int i = 0; // Apply "b58 = b58 * 256 + ch". for (std::vector<unsigned char>::reverse_iterator it = b58.rbegin(); (carry != 0 || i < length) && (it != b58.rend()); it++, i++) { @@ -109,7 +109,7 @@ std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) assert(carry == 0); length = i; - pbegin++; + input = input.subspan(1); } // Skip leading zeroes in base58 result. std::vector<unsigned char>::iterator it = b58.begin() + (size - length); @@ -124,11 +124,6 @@ std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) return str; } -std::string EncodeBase58(const std::vector<unsigned char>& vch) -{ - return EncodeBase58(vch.data(), vch.data() + vch.size()); -} - bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len) { if (!ValidAsCString(str)) { @@ -137,10 +132,10 @@ bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, in return DecodeBase58(str.c_str(), vchRet, max_ret_len); } -std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn) +std::string EncodeBase58Check(Span<const unsigned char> input) { // add 4-byte hash check to the end - std::vector<unsigned char> vch(vchIn); + std::vector<unsigned char> vch(input.begin(), input.end()); uint256 hash = Hash(vch); vch.insert(vch.end(), (unsigned char*)&hash, (unsigned char*)&hash + 4); return EncodeBase58(vch); diff --git a/src/base58.h b/src/base58.h index a869ee1e75..468c3e2589 100644 --- a/src/base58.h +++ b/src/base58.h @@ -15,20 +15,15 @@ #define BITCOIN_BASE58_H #include <attributes.h> +#include <span.h> #include <string> #include <vector> /** - * Encode a byte sequence as a base58-encoded string. - * pbegin and pend cannot be nullptr, unless both are. + * Encode a byte span as a base58-encoded string */ -std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend); - -/** - * Encode a byte vector as a base58-encoded string - */ -std::string EncodeBase58(const std::vector<unsigned char>& vch); +std::string EncodeBase58(Span<const unsigned char> input); /** * Decode a base58-encoded string (str) into a byte vector (vchRet). @@ -37,9 +32,9 @@ std::string EncodeBase58(const std::vector<unsigned char>& vch); NODISCARD bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); /** - * Encode a byte vector into a base58-encoded string, including checksum + * Encode a byte span into a base58-encoded string, including checksum */ -std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn); +std::string EncodeBase58Check(Span<const unsigned char> input); /** * Decode a base58-encoded string (str) that includes a checksum into a byte diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 00544cba31..18cb5de196 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -20,7 +20,7 @@ static void Base58Encode(benchmark::Bench& bench) } }; bench.batch(buff.size()).unit("byte").run([&] { - EncodeBase58(buff.data(), buff.data() + buff.size()); + EncodeBase58(buff); }); } @@ -34,10 +34,8 @@ static void Base58CheckEncode(benchmark::Bench& bench) 200, 24 } }; - std::vector<unsigned char> vch; - vch.assign(buff.begin(), buff.end()); bench.batch(buff.size()).unit("byte").run([&] { - EncodeBase58Check(vch); + EncodeBase58Check(buff); }); } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index b04cc12059..227626f40f 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -50,28 +50,23 @@ static bool AppInit(int argc, char* argv[]) util::ThreadSetInternalName("init"); - // - // Parameters - // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() SetupServerArgs(node); + ArgsManager& args = *Assert(node.args); std::string error; - if (!gArgs.ParseParameters(argc, argv, error)) { + if (!args.ParseParameters(argc, argv, error)) { return InitError(Untranslated(strprintf("Error parsing command line arguments: %s\n", error))); } // Process help and version before taking care about datadir - if (HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; - if (gArgs.IsArgSet("-version")) - { + if (args.IsArgSet("-version")) { strUsage += FormatParagraph(LicenseInfo()) + "\n"; - } - else - { + } else { strUsage += "\nUsage: bitcoind [options] Start " PACKAGE_NAME "\n"; - strUsage += "\n" + gArgs.GetHelpMessage(); + strUsage += "\n" + args.GetHelpMessage(); } tfm::format(std::cout, "%s", strUsage); @@ -82,14 +77,14 @@ static bool AppInit(int argc, char* argv[]) try { if (!CheckDataDirOption()) { - return InitError(Untranslated(strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")))); + return InitError(Untranslated(strprintf("Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")))); } - if (!gArgs.ReadConfigFiles(error, true)) { + if (!args.ReadConfigFiles(error, true)) { return InitError(Untranslated(strprintf("Error reading configuration file: %s\n", error))); } // Check for -chain, -testnet or -regtest parameter (Params() calls are only valid after this clause) try { - SelectParams(gArgs.GetChainName()); + SelectParams(args.GetChainName()); } catch (const std::exception& e) { return InitError(Untranslated(strprintf("%s\n", e.what()))); } @@ -101,23 +96,21 @@ static bool AppInit(int argc, char* argv[]) } } - if (!gArgs.InitSettings(error)) { + if (!args.InitSettings(error)) { InitError(Untranslated(error)); return false; } // -server defaults to true for bitcoind but not for the GUI so do this here - gArgs.SoftSetBoolArg("-server", true); + args.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console - InitLogging(); - InitParameterInteraction(); - if (!AppInitBasicSetup()) - { + InitLogging(args); + InitParameterInteraction(args); + if (!AppInitBasicSetup(args)) { // InitError will have been called with detailed error, which ends up on console return false; } - if (!AppInitParameterInteraction()) - { + if (!AppInitParameterInteraction(args)) { // InitError will have been called with detailed error, which ends up on console return false; } @@ -126,8 +119,7 @@ static bool AppInit(int argc, char* argv[]) // InitError will have been called with detailed error, which ends up on console return false; } - if (gArgs.GetBoolArg("-daemon", false)) - { + if (args.GetBoolArg("-daemon", false)) { #if HAVE_DECL_DAEMON #if defined(MAC_OSX) #pragma GCC diagnostic push diff --git a/src/chain.h b/src/chain.h index 802e23f775..43e8a39f36 100644 --- a/src/chain.h +++ b/src/chain.h @@ -398,12 +398,6 @@ public: return vChain[nHeight]; } - /** Compare two chains efficiently. */ - friend bool operator==(const CChain &a, const CChain &b) { - return a.vChain.size() == b.vChain.size() && - a.vChain[a.vChain.size() - 1] == b.vChain[b.vChain.size() - 1]; - } - /** Efficiently check whether a block is present in this chain. */ bool Contains(const CBlockIndex *pindex) const { return (*this)[pindex->nHeight] == pindex; diff --git a/src/compat.h b/src/compat.h index 68f6eb692c..0be02cae03 100644 --- a/src/compat.h +++ b/src/compat.h @@ -11,9 +11,6 @@ #endif #ifdef WIN32 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN 1 -#endif #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 18dc7a69e2..380d4eb8ac 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -35,6 +35,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-discardfee=<amt>", "-fallbackfee=<amt>", "-keypool=<n>", + "-maxapsfee=<n>", "-maxtxfee=<amt>", "-mintxfee=<amt>", "-paytxfee=<amt>", diff --git a/src/hash.cpp b/src/hash.cpp index 4c09f5f646..83b90ae063 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -77,3 +77,10 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he num[3] = (nChild >> 0) & 0xFF; CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } + +uint256 SHA256Uint256(const uint256& input) +{ + uint256 result; + CSHA256().Write(input.begin(), 32).Finalize(result.begin()); + return result; +} diff --git a/src/hash.h b/src/hash.h index 71806483ff..c16bbb48ce 100644 --- a/src/hash.h +++ b/src/hash.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_HASH_H #define BITCOIN_HASH_H +#include <attributes.h> #include <crypto/common.h> #include <crypto/ripemd160.h> #include <crypto/sha256.h> @@ -98,7 +99,7 @@ inline uint160 Hash160(const T1& in1) class CHashWriter { private: - CHash256 ctx; + CSHA256 ctx; const int nType; const int nVersion; @@ -110,13 +111,27 @@ public: int GetVersion() const { return nVersion; } void write(const char *pch, size_t size) { - ctx.Write({(const unsigned char*)pch, size}); + ctx.Write((const unsigned char*)pch, size); } - // invalidates the object + /** Compute the double-SHA256 hash of all data written to this object. + * + * Invalidates this object. + */ uint256 GetHash() { uint256 result; - ctx.Finalize(result); + ctx.Finalize(result.begin()); + ctx.Reset().Write(result.begin(), CSHA256::OUTPUT_SIZE).Finalize(result.begin()); + return result; + } + + /** Compute the SHA256 hash of all data written to this object. + * + * Invalidates this object. + */ + uint256 GetSHA256() { + uint256 result; + ctx.Finalize(result.begin()); return result; } @@ -124,9 +139,8 @@ public: * Returns the first 64 bits from the resulting hash. */ inline uint64_t GetCheapHash() { - unsigned char result[CHash256::OUTPUT_SIZE]; - ctx.Finalize(result); - return ReadLE64(result); + uint256 result = GetHash(); + return ReadLE64(result.begin()); } template<typename T> @@ -181,6 +195,9 @@ uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL return ss.GetHash(); } +/** Single-SHA256 a 32-byte input (represented as uint256). */ +NODISCARD uint256 SHA256Uint256(const uint256& input); + unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vDataToHash); void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]); diff --git a/src/index/base.cpp b/src/index/base.cpp index f587205a28..e67b813763 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -319,3 +319,12 @@ void BaseIndex::Stop() m_thread_sync.join(); } } + +IndexSummary BaseIndex::GetSummary() const +{ + IndexSummary summary{}; + summary.name = GetName(); + summary.synced = m_synced; + summary.best_block_height = m_best_block_index.load()->nHeight; + return summary; +} diff --git a/src/index/base.h b/src/index/base.h index 3fab810bb2..8559e3cb64 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -13,6 +13,12 @@ class CBlockIndex; +struct IndexSummary { + std::string name; + bool synced{false}; + int best_block_height{0}; +}; + /** * Base class for indices of blockchain data. This implements * CValidationInterface and ensures blocks are indexed sequentially according @@ -21,6 +27,13 @@ class CBlockIndex; class BaseIndex : public CValidationInterface { protected: + /** + * The database stores a block locator of the chain the database is synced to + * so that the index can efficiently determine the point it last stopped at. + * A locator is used instead of a simple hash of the chain tip because blocks + * and block index entries may not be flushed to disk until after this database + * is updated. + */ class DB : public CDBWrapper { public: @@ -106,6 +119,9 @@ public: /// Stops the instance from staying in sync with blockchain updates. void Stop(); + + /// Get a summary of the index and its state. + IndexSummary GetSummary() const; }; #endif // BITCOIN_INDEX_BASE_H diff --git a/src/index/disktxpos.h b/src/index/disktxpos.h new file mode 100644 index 0000000000..69696b0ec5 --- /dev/null +++ b/src/index/disktxpos.h @@ -0,0 +1,35 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_DISKTXPOS_H +#define BITCOIN_INDEX_DISKTXPOS_H + +#include <flatfile.h> +#include <serialize.h> + +struct CDiskTxPos : public FlatFilePos +{ + unsigned int nTxOffset; // after header + + SERIALIZE_METHODS(CDiskTxPos, obj) + { + READWRITEAS(FlatFilePos, obj); + READWRITE(VARINT(obj.nTxOffset)); + } + + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + } + + CDiskTxPos() { + SetNull(); + } + + void SetNull() { + FlatFilePos::SetNull(); + nTxOffset = 0; + } +}; + + +#endif // BITCOIN_INDEX_DISKTXPOS_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 64472714cc..462ac5962f 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <index/disktxpos.h> #include <index/txindex.h> #include <node/ui_interface.h> #include <shutdown.h> @@ -15,38 +16,9 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -struct CDiskTxPos : public FlatFilePos -{ - unsigned int nTxOffset; // after header - - SERIALIZE_METHODS(CDiskTxPos, obj) - { - READWRITEAS(FlatFilePos, obj); - READWRITE(VARINT(obj.nTxOffset)); - } - - CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - CDiskTxPos() { - SetNull(); - } - void SetNull() { - FlatFilePos::SetNull(); - nTxOffset = 0; - } -}; - -/** - * Access to the txindex database (indexes/txindex/) - * - * The database stores a block locator of the chain the database is synced to - * so that the TxIndex can efficiently determine the point it last stopped at. - * A locator is used instead of a simple hash of the chain tip because blocks - * and block index entries may not be flushed to disk until after this database - * is updated. - */ +/** Access to the txindex database (indexes/txindex/) */ class TxIndex::DB : public BaseIndex::DB { public: diff --git a/src/init.cpp b/src/init.cpp index f5aef08211..ecd57960ad 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -107,14 +107,14 @@ static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; */ static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; -static fs::path GetPidFile() +static fs::path GetPidFile(const ArgsManager& args) { - return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); + return AbsPathForConfigVal(fs::path(args.GetArg("-pid", BITCOIN_PID_FILENAME))); } -NODISCARD static bool CreatePidFile() +NODISCARD static bool CreatePidFile(const ArgsManager& args) { - fsbridge::ofstream file{GetPidFile()}; + fsbridge::ofstream file{GetPidFile(args)}; if (file) { #ifdef WIN32 tfm::format(file, "%d\n", GetCurrentProcessId()); @@ -123,7 +123,7 @@ NODISCARD static bool CreatePidFile() #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile(args).string(), std::strerror(errno))); } } @@ -180,6 +180,7 @@ void Shutdown(NodeContext& node) TRY_LOCK(g_shutdown_mutex, lock_shutdown); if (!lock_shutdown) return; LogPrintf("%s: In progress...\n", __func__); + Assert(node.args); /// Note: Shutdown() must be able to handle cases in which initialization failed part of the way, /// for example if the data directory was found to be locked. @@ -230,7 +231,7 @@ void Shutdown(NodeContext& node) node.connman.reset(); node.banman.reset(); - if (::mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + if (::mempool.IsLoaded() && node.args->GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(::mempool); } @@ -301,19 +302,19 @@ void Shutdown(NodeContext& node) GetMainSignals().UnregisterBackgroundSignalScheduler(); globalVerifyHandle.reset(); ECC_Stop(); - node.args = nullptr; node.mempool = nullptr; node.chainman = nullptr; node.scheduler.reset(); try { - if (!fs::remove(GetPidFile())) { + if (!fs::remove(GetPidFile(*node.args))) { LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); } } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } + node.args = nullptr; LogPrintf("%s: done\n", __func__); } @@ -372,7 +373,7 @@ void SetupServerArgs(NodeContext& node) node.args = &gArgs; ArgsManager& argsman = *node.args; - SetupHelpOptions(gArgs); + SetupHelpOptions(argsman); argsman.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); @@ -599,21 +600,6 @@ std::string LicenseInfo() "\n"; } -#if HAVE_SYSTEM -static void BlockNotifyCallback(SynchronizationState sync_state, const CBlockIndex* pBlockIndex) -{ - if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) - return; - - std::string strCmd = gArgs.GetArg("-blocknotify", ""); - if (!strCmd.empty()) { - boost::replace_all(strCmd, "%s", pBlockIndex->GetBlockHash().GetHex()); - std::thread t(runCommand, strCmd); - t.detach(); // thread runs free - } -} -#endif - static bool fHaveGenesis = false; static Mutex g_genesis_wait_mutex; static std::condition_variable g_genesis_wait_cv; @@ -684,7 +670,7 @@ static void CleanupBlockRevFiles() } } -static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles) +static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) { const CChainParams& chainparams = Params(); ScheduleBatchPriority(); @@ -746,13 +732,13 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp } } - if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { + if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); StartShutdown(); return; } } // End scope of CImportingNow - if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { LoadMempool(::mempool); } ::mempool.SetIsLoaded(!ShutdownRequested()); @@ -780,6 +766,7 @@ static bool InitSanityCheck() static bool AppInitServers(const util::Ref& context, NodeContext& node) { + const ArgsManager& args = *Assert(node.args); RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) @@ -788,71 +775,71 @@ static bool AppInitServers(const util::Ref& context, NodeContext& node) node.rpc_interruption_point = RpcInterruptionPoint; if (!StartHTTPRPC(context)) return false; - if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); + if (args.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); StartHTTPServer(); return true; } // Parameter interaction based on rules -void InitParameterInteraction() +void InitParameterInteraction(ArgsManager& args) { // when specifying an explicit binding address, you want to listen on it // even when -connect or -proxy is specified - if (gArgs.IsArgSet("-bind")) { - if (gArgs.SoftSetBoolArg("-listen", true)) + if (args.IsArgSet("-bind")) { + if (args.SoftSetBoolArg("-listen", true)) LogPrintf("%s: parameter interaction: -bind set -> setting -listen=1\n", __func__); } - if (gArgs.IsArgSet("-whitebind")) { - if (gArgs.SoftSetBoolArg("-listen", true)) + if (args.IsArgSet("-whitebind")) { + if (args.SoftSetBoolArg("-listen", true)) LogPrintf("%s: parameter interaction: -whitebind set -> setting -listen=1\n", __func__); } - if (gArgs.IsArgSet("-connect")) { + if (args.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen by default - if (gArgs.SoftSetBoolArg("-dnsseed", false)) + if (args.SoftSetBoolArg("-dnsseed", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -dnsseed=0\n", __func__); - if (gArgs.SoftSetBoolArg("-listen", false)) + if (args.SoftSetBoolArg("-listen", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -listen=0\n", __func__); } - if (gArgs.IsArgSet("-proxy")) { + if (args.IsArgSet("-proxy")) { // to protect privacy, do not listen by default if a default proxy server is specified - if (gArgs.SoftSetBoolArg("-listen", false)) + if (args.SoftSetBoolArg("-listen", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -listen=0\n", __func__); // to protect privacy, do not use UPNP when a proxy is set. The user may still specify -listen=1 // to listen locally, so don't rely on this happening through -listen below. - if (gArgs.SoftSetBoolArg("-upnp", false)) + if (args.SoftSetBoolArg("-upnp", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -upnp=0\n", __func__); // to protect privacy, do not discover addresses by default - if (gArgs.SoftSetBoolArg("-discover", false)) + if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -discover=0\n", __func__); } - if (!gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { + if (!args.GetBoolArg("-listen", DEFAULT_LISTEN)) { // do not map ports or try to retrieve public IP when not listening (pointless) - if (gArgs.SoftSetBoolArg("-upnp", false)) + if (args.SoftSetBoolArg("-upnp", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -upnp=0\n", __func__); - if (gArgs.SoftSetBoolArg("-discover", false)) + if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -discover=0\n", __func__); - if (gArgs.SoftSetBoolArg("-listenonion", false)) + if (args.SoftSetBoolArg("-listenonion", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -listenonion=0\n", __func__); } - if (gArgs.IsArgSet("-externalip")) { + if (args.IsArgSet("-externalip")) { // if an explicit public IP is specified, do not try to find others - if (gArgs.SoftSetBoolArg("-discover", false)) + if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -externalip set -> setting -discover=0\n", __func__); } // disable whitelistrelay in blocksonly mode - if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { - if (gArgs.SoftSetBoolArg("-whitelistrelay", false)) + if (args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { + if (args.SoftSetBoolArg("-whitelistrelay", false)) LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -whitelistrelay=0\n", __func__); } // Forcing relay from whitelisted hosts implies we will accept relays from them in the first place. - if (gArgs.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { - if (gArgs.SoftSetBoolArg("-whitelistrelay", true)) + if (args.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { + if (args.SoftSetBoolArg("-whitelistrelay", true)) LogPrintf("%s: parameter interaction: -whitelistforcerelay=1 -> setting -whitelistrelay=1\n", __func__); } } @@ -863,18 +850,18 @@ void InitParameterInteraction() * Note that this is called very early in the process lifetime, so you should be * careful about what global state you rely on here. */ -void InitLogging() +void InitLogging(const ArgsManager& args) { - LogInstance().m_print_to_file = !gArgs.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(gArgs.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); - LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", !gArgs.GetBoolArg("-daemon", false)); - LogInstance().m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); - LogInstance().m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); + LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); + LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); + LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); + LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); #ifdef HAVE_THREAD_LOCAL - LogInstance().m_log_threadnames = gArgs.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); + LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); #endif - fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); + fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); std::string version_string = FormatFullVersion(); #ifdef DEBUG @@ -909,7 +896,7 @@ std::set<BlockFilterType> g_enabled_filter_types; std::terminate(); }; -bool AppInitBasicSetup() +bool AppInitBasicSetup(ArgsManager& args) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -929,7 +916,7 @@ bool AppInitBasicSetup() } #ifndef WIN32 - if (!gArgs.GetBoolArg("-sysperms", false)) { + if (!args.GetBoolArg("-sysperms", false)) { umask(077); } @@ -951,7 +938,7 @@ bool AppInitBasicSetup() return true; } -bool AppInitParameterInteraction() +bool AppInitParameterInteraction(const ArgsManager& args) { const CChainParams& chainparams = Params(); // ********************************************************* Step 2: parameter interactions @@ -961,9 +948,9 @@ bool AppInitParameterInteraction() // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden // on the command line or in this network's section of the config file. - std::string network = gArgs.GetChainName(); + std::string network = args.GetChainName(); bilingual_str errors; - for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { + for (const auto& arg : args.GetUnsuitableSectionOnlyArgs()) { errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, network, network); } @@ -973,7 +960,7 @@ bool AppInitParameterInteraction() // Warn if unrecognized section name are present in the config file. bilingual_str warnings; - for (const auto& section : gArgs.GetUnrecognizedSections()) { + for (const auto& section : args.GetUnrecognizedSections()) { warnings += strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized.") + Untranslated("\n"), section.m_file, section.m_line, section.m_name); } @@ -982,15 +969,15 @@ bool AppInitParameterInteraction() } if (!fs::is_directory(GetBlocksDir())) { - return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", ""))); + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), args.GetArg("-blocksdir", ""))); } // parse and validate enabled filter types - std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX); + std::string blockfilterindex_value = args.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX); if (blockfilterindex_value == "" || blockfilterindex_value == "1") { g_enabled_filter_types = AllBlockFilterTypes(); } else if (blockfilterindex_value != "0") { - const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex"); + const std::vector<std::string> names = args.GetArgs("-blockfilterindex"); for (const auto& name : names) { BlockFilterType filter_type; if (!BlockFilterTypeByName(name, filter_type)) { @@ -1001,7 +988,7 @@ bool AppInitParameterInteraction() } // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled. - if (gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)) { + if (args.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)) { if (g_enabled_filter_types.count(BlockFilterType::BASIC) != 1) { return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); } @@ -1010,8 +997,8 @@ bool AppInitParameterInteraction() } // if using block pruning, then disallow txindex - if (gArgs.GetArg("-prune", 0)) { - if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) + if (args.GetArg("-prune", 0)) { + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); if (!g_enabled_filter_types.empty()) { return InitError(_("Prune mode is incompatible with -blockfilterindex.")); @@ -1019,14 +1006,14 @@ bool AppInitParameterInteraction() } // -bind and -whitebind can't be set when not listening - size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size(); - if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { + size_t nUserBind = args.GetArgs("-bind").size() + args.GetArgs("-whitebind").size(); + if (nUserBind != 0 && !args.GetBoolArg("-listen", DEFAULT_LISTEN)) { return InitError(Untranslated("Cannot set -bind or -whitebind together with -listen=0")); } // Make sure enough file descriptors are available int nBind = std::max(nUserBind, size_t(1)); - nUserMaxConnections = gArgs.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); + nUserMaxConnections = args.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); // Trim requested connection counts, to fit into system limitations @@ -1046,9 +1033,9 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (gArgs.IsArgSet("-debug")) { + if (args.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string> categories = gArgs.GetArgs("-debug"); + const std::vector<std::string> categories = args.GetArgs("-debug"); if (std::none_of(categories.begin(), categories.end(), [](std::string cat){return cat == "0" || cat == "none";})) { @@ -1061,28 +1048,28 @@ bool AppInitParameterInteraction() } // Now remove the logging categories which were explicitly excluded - for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { + for (const std::string& cat : args.GetArgs("-debugexclude")) { if (!LogInstance().DisableCategory(cat)) { InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); } } // Checkmempool and checkblockindex default to true in regtest mode - int ratio = std::min<int>(std::max<int>(gArgs.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); + int ratio = std::min<int>(std::max<int>(args.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); if (ratio != 0) { mempool.setSanityCheck(1.0 / ratio); } - fCheckBlockIndex = gArgs.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); - fCheckpointsEnabled = gArgs.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); + fCheckBlockIndex = args.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); + fCheckpointsEnabled = args.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); - hashAssumeValid = uint256S(gArgs.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); + hashAssumeValid = uint256S(args.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); if (!hashAssumeValid.IsNull()) LogPrintf("Assuming ancestors of block %s have valid signatures.\n", hashAssumeValid.GetHex()); else LogPrintf("Validating signatures for all blocks.\n"); - if (gArgs.IsArgSet("-minimumchainwork")) { - const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); + if (args.IsArgSet("-minimumchainwork")) { + const std::string minChainWorkStr = args.GetArg("-minimumchainwork", ""); if (!IsHexNumber(minChainWorkStr)) { return InitError(strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), minChainWorkStr)); } @@ -1096,22 +1083,21 @@ bool AppInitParameterInteraction() } // mempool limits - int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; + int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMin = args.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // incremental relay fee sets the minimum feerate increase necessary for BIP 125 replacement in the mempool // and the amount the mempool min fee increases above the feerate of txs evicted due to mempool limiting. - if (gArgs.IsArgSet("-incrementalrelayfee")) - { + if (args.IsArgSet("-incrementalrelayfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-incrementalrelayfee", ""), n)) - return InitError(AmountErrMsg("incrementalrelayfee", gArgs.GetArg("-incrementalrelayfee", ""))); + if (!ParseMoney(args.GetArg("-incrementalrelayfee", ""), n)) + return InitError(AmountErrMsg("incrementalrelayfee", args.GetArg("-incrementalrelayfee", ""))); incrementalRelayFee = CFeeRate(n); } // block pruning; get the amount of disk space (in MiB) to allot for block & undo files - int64_t nPruneArg = gArgs.GetArg("-prune", 0); + int64_t nPruneArg = args.GetArg("-prune", 0); if (nPruneArg < 0) { return InitError(_("Prune cannot be configured with a negative value.")); } @@ -1128,20 +1114,20 @@ bool AppInitParameterInteraction() fPruneMode = true; } - nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); + nConnectTimeout = args.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; } - peer_connect_timeout = gArgs.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); + peer_connect_timeout = args.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); } - if (gArgs.IsArgSet("-minrelaytxfee")) { + if (args.IsArgSet("-minrelaytxfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-minrelaytxfee", ""), n)) { - return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", ""))); + if (!ParseMoney(args.GetArg("-minrelaytxfee", ""), n)) { + return InitError(AmountErrMsg("minrelaytxfee", args.GetArg("-minrelaytxfee", ""))); } // High fee check is done afterward in CWallet::CreateWalletFromFile() ::minRelayTxFee = CFeeRate(n); @@ -1153,48 +1139,46 @@ bool AppInitParameterInteraction() // Sanity check argument for min fee for including tx in block // TODO: Harmonize which arguments need sanity checking and where that happens - if (gArgs.IsArgSet("-blockmintxfee")) - { + if (args.IsArgSet("-blockmintxfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-blockmintxfee", ""), n)) - return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", ""))); + if (!ParseMoney(args.GetArg("-blockmintxfee", ""), n)) + return InitError(AmountErrMsg("blockmintxfee", args.GetArg("-blockmintxfee", ""))); } // Feerate used to define dust. Shouldn't be changed lightly as old // implementations may inadvertently create non-standard transactions - if (gArgs.IsArgSet("-dustrelayfee")) - { + if (args.IsArgSet("-dustrelayfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-dustrelayfee", ""), n)) - return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", ""))); + if (!ParseMoney(args.GetArg("-dustrelayfee", ""), n)) + return InitError(AmountErrMsg("dustrelayfee", args.GetArg("-dustrelayfee", ""))); dustRelayFee = CFeeRate(n); } - fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); + fRequireStandard = !args.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !fRequireStandard) { return InitError(strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString())); } - nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp); + nBytesPerSigOp = args.GetArg("-bytespersigop", nBytesPerSigOp); if (!g_wallet_init_interface.ParameterInteraction()) return false; - fIsBareMultisigStd = gArgs.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); - fAcceptDatacarrier = gArgs.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); - nMaxDatacarrierBytes = gArgs.GetArg("-datacarriersize", nMaxDatacarrierBytes); + fIsBareMultisigStd = args.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); + fAcceptDatacarrier = args.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); + nMaxDatacarrierBytes = args.GetArg("-datacarriersize", nMaxDatacarrierBytes); // Option to startup with mocktime set (used for regression testing): - SetMockTime(gArgs.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op + SetMockTime(args.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op - if (gArgs.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) + if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) + if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) return InitError(Untranslated("rpcserialversion must be non-negative.")); - if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) + if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - nMaxTipAge = gArgs.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); + nMaxTipAge = args.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); return true; } @@ -1247,14 +1231,15 @@ bool AppInitLockDataDirectory() bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { + const ArgsManager& args = *Assert(node.args); const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization - if (!CreatePidFile()) { + if (!CreatePidFile(args)) { // Detailed error printed inside CreatePidFile(). return false; } if (LogInstance().m_print_to_file) { - if (gArgs.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { + if (args.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { // Do this first since it both loads a bunch of debug.log into memory, // and because this needs to happen before any other debug.log printing LogInstance().ShrinkDebugFile(); @@ -1271,10 +1256,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA LogPrintf("Using data directory %s\n", GetDataDir().string()); // Only log conf file usage message if conf file actually exists. - fs::path config_file_path = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)); + fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); if (fs::exists(config_file_path)) { LogPrintf("Config file: %s\n", config_file_path.string()); - } else if (gArgs.IsArgSet("-conf")) { + } else if (args.IsArgSet("-conf")) { // Warn if no conf file exists at path provided by user InitWarning(strprintf(_("The specified config file %s does not exist\n"), config_file_path.string())); } else { @@ -1283,23 +1268,23 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } // Log the config arguments to debug.log - gArgs.LogArgs(); + args.LogArgs(); LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. - if (gArgs.IsArgSet("-datadir") && !fs::path(gArgs.GetArg("-datadir", "")).is_absolute()) { + if (args.IsArgSet("-datadir") && !fs::path(args.GetArg("-datadir", "")).is_absolute()) { LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " /* Continued */ "current working directory '%s'. This is fragile, because if bitcoin is started in the future " "from a different location, it will be unable to locate the current data files. There could " "also be data loss if bitcoin is started while in a temporary directory.\n", - gArgs.GetArg("-datadir", ""), fs::current_path().string()); + args.GetArg("-datadir", ""), fs::current_path().string()); } InitSignatureCache(); InitScriptExecutionCache(); - int script_threads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + int script_threads = args.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (script_threads <= 0) { // -par=0 means autodetect (number of cores - 1 script threads) // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) @@ -1355,8 +1340,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA * that the server is there and will be ready later). Warmup mode will * be disabled when initialisation is finished. */ - if (gArgs.GetBoolArg("-server", false)) - { + if (args.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); if (!AppInitServers(context, node)) return InitError(_("Unable to start HTTP server. See debug log for details.")); @@ -1376,9 +1360,9 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // need to reindex later. assert(!node.banman); - node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, args.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); - node.connman = MakeUnique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), gArgs.GetBoolArg("-networkactive", true)); + node.connman = MakeUnique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), args.GetBoolArg("-networkactive", true)); // Make mempool generally available in the node context. For example the connection manager, wallet, or RPC threads, // which are all started after this, may use it from the node context. assert(!node.mempool); @@ -1387,12 +1371,12 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA node.chainman = &g_chainman; ChainstateManager& chainman = *Assert(node.chainman); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, chainman, *node.mempool)); + node.peer_logic.reset(new PeerLogicValidation(*node.connman, node.banman.get(), *node.scheduler, chainman, *node.mempool)); RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; - for (const std::string& cmt : gArgs.GetArgs("-uacomment")) { + for (const std::string& cmt : args.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(cmt); @@ -1403,9 +1387,9 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } - if (gArgs.IsArgSet("-onlynet")) { + if (args.IsArgSet("-onlynet")) { std::set<enum Network> nets; - for (const std::string& snet : gArgs.GetArgs("-onlynet")) { + for (const std::string& snet : args.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); @@ -1419,12 +1403,12 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } // Check for host lookup allowed before parsing any network related parameters - fNameLookup = gArgs.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); + fNameLookup = args.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); - bool proxyRandomize = gArgs.GetBoolArg("-proxyrandomize", DEFAULT_PROXYRANDOMIZE); + bool proxyRandomize = args.GetBoolArg("-proxyrandomize", DEFAULT_PROXYRANDOMIZE); // -proxy sets a proxy for all outgoing network traffic // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default - std::string proxyArg = gArgs.GetArg("-proxy", ""); + std::string proxyArg = args.GetArg("-proxy", ""); SetReachable(NET_ONION, false); if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; @@ -1446,7 +1430,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // -onion can be used to set only a proxy for .onion, or override normal proxy for .onion addresses // -noonion (or -onion=0) disables connecting to .onion entirely // An empty string is used to not override the onion proxy (in which case it defaults to -proxy set above, or none) - std::string onionArg = gArgs.GetArg("-onion", ""); + std::string onionArg = args.GetArg("-onion", ""); if (onionArg != "") { if (onionArg == "0") { // Handle -noonion/-onion=0 SetReachable(NET_ONION, false); @@ -1464,11 +1448,11 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } // see Step 2: parameter interactions for more information about these - fListen = gArgs.GetBoolArg("-listen", DEFAULT_LISTEN); - fDiscover = gArgs.GetBoolArg("-discover", true); - g_relay_txes = !gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); + fListen = args.GetBoolArg("-listen", DEFAULT_LISTEN); + fDiscover = args.GetBoolArg("-discover", true); + g_relay_txes = !args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); - for (const std::string& strAddr : gArgs.GetArgs("-externalip")) { + for (const std::string& strAddr : args.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); @@ -1477,8 +1461,8 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } // Read asmap file if configured - if (gArgs.IsArgSet("-asmap")) { - fs::path asmap_path = fs::path(gArgs.GetArg("-asmap", "")); + if (args.IsArgSet("-asmap")) { + fs::path asmap_path = fs::path(args.GetArg("-asmap", "")); if (asmap_path.empty()) { asmap_path = DEFAULT_ASMAP_FILENAME; } @@ -1511,22 +1495,22 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA uint64_t nMaxOutboundLimit = 0; //unlimited unless -maxuploadtarget is set uint64_t nMaxOutboundTimeframe = MAX_UPLOAD_TIMEFRAME; - if (gArgs.IsArgSet("-maxuploadtarget")) { - nMaxOutboundLimit = gArgs.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET)*1024*1024; + if (args.IsArgSet("-maxuploadtarget")) { + nMaxOutboundLimit = args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET) * 1024 * 1024; } // ********************************************************* Step 7: load block chain - fReindex = gArgs.GetBoolArg("-reindex", false); - bool fReindexChainState = gArgs.GetBoolArg("-reindex-chainstate", false); + fReindex = args.GetBoolArg("-reindex", false); + bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); // cache size calculations - int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20); + int64_t nTotalCache = (args.GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); nTotalCache -= nBlockTreeDBCache; - int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); + int64_t nTxIndexCache = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); nTotalCache -= nTxIndexCache; int64_t filter_index_cache = 0; if (!g_enabled_filter_types.empty()) { @@ -1539,10 +1523,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache - int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); - if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); } for (BlockFilterType filter_type : g_enabled_filter_types) { @@ -1707,7 +1691,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { uiInterface.InitMessage(_("Verifying blocks...").translated); - if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + if (fHavePruned && args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", MIN_BLOCKS_TO_KEEP); } @@ -1725,10 +1709,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // Only verify the DB of the active chainstate. This is fixed in later // work when we allow VerifyDB to be parameterized by chainstate. if (&::ChainstateActive() == chainstate && - !CVerifyDB().VerifyDB( + !CVerifyDB().VerifyDB( chainparams, &chainstate->CoinsDB(), - gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); failed_verification = true; break; @@ -1784,7 +1768,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA fFeeEstimatesInitialized = true; // ********************************************************* Step 8: start indexers - if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { g_txindex = MakeUnique<TxIndex>(nTxIndexCache, false, fReindex); g_txindex->Start(); } @@ -1844,16 +1828,31 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } #if HAVE_SYSTEM - if (gArgs.IsArgSet("-blocknotify")) + if (args.IsArgSet("-blocknotify")) { + const std::string block_notify = args.GetArg("-blocknotify", ""); + const auto BlockNotifyCallback = [block_notify](SynchronizationState sync_state, const CBlockIndex* pBlockIndex) { + if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) + return; + + std::string strCmd = block_notify; + if (!strCmd.empty()) { + boost::replace_all(strCmd, "%s", pBlockIndex->GetBlockHash().GetHex()); + std::thread t(runCommand, strCmd); + t.detach(); // thread runs free + } + }; uiInterface.NotifyBlockTip_connect(BlockNotifyCallback); + } #endif std::vector<fs::path> vImportFiles; - for (const std::string& strFile : gArgs.GetArgs("-loadblock")) { + for (const std::string& strFile : args.GetArgs("-loadblock")) { vImportFiles.push_back(strFile); } - g_load_block = std::thread(&TraceThread<std::function<void()>>, "loadblk", [=, &chainman]{ ThreadImport(chainman, vImportFiles); }); + g_load_block = std::thread(&TraceThread<std::function<void()>>, "loadblk", [=, &chainman, &args] { + ThreadImport(chainman, vImportFiles, args); + }); // Wait for genesis block to be processed { @@ -1892,13 +1891,13 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } LogPrintf("nBestHeight = %d\n", chain_active_height); - if (gArgs.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) + if (args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) StartTorControl(); Discover(); // Map ports with UPnP - if (gArgs.GetBoolArg("-upnp", DEFAULT_UPNP)) { + if (args.GetBoolArg("-upnp", DEFAULT_UPNP)) { StartMapPort(); } @@ -1913,41 +1912,41 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peer_logic.get(); - connOptions.nSendBufferMaxSize = 1000*gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); - connOptions.nReceiveFloodSize = 1000*gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); - connOptions.m_added_nodes = gArgs.GetArgs("-addnode"); + connOptions.nSendBufferMaxSize = 1000 * args.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); + connOptions.nReceiveFloodSize = 1000 * args.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); + connOptions.m_added_nodes = args.GetArgs("-addnode"); connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; connOptions.m_peer_connect_timeout = peer_connect_timeout; - for (const std::string& strBind : gArgs.GetArgs("-bind")) { + for (const std::string& strBind : args.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind, addrBind, GetListenPort(), false)) { return InitError(ResolveErrMsg("bind", strBind)); } connOptions.vBinds.push_back(addrBind); } - for (const std::string& strBind : gArgs.GetArgs("-whitebind")) { + for (const std::string& strBind : args.GetArgs("-whitebind")) { NetWhitebindPermissions whitebind; bilingual_str error; if (!NetWhitebindPermissions::TryParse(strBind, whitebind, error)) return InitError(error); connOptions.vWhiteBinds.push_back(whitebind); } - for (const auto& net : gArgs.GetArgs("-whitelist")) { + for (const auto& net : args.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; bilingual_str error; if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(error); connOptions.vWhitelistedRange.push_back(subnet); } - connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); + connOptions.vSeedNodes = args.GetArgs("-seednode"); // Initiate outbound connections unless connect=0 - connOptions.m_use_addrman_outgoing = !gArgs.IsArgSet("-connect"); + connOptions.m_use_addrman_outgoing = !args.IsArgSet("-connect"); if (!connOptions.m_use_addrman_outgoing) { - const auto connect = gArgs.GetArgs("-connect"); + const auto connect = args.GetArgs("-connect"); if (connect.size() != 1 || connect[0] != "0") { connOptions.m_specified_outgoing = connect; } diff --git a/src/init.h b/src/init.h index 20008ba5be..ce12a80dc7 100644 --- a/src/init.h +++ b/src/init.h @@ -8,8 +8,8 @@ #include <memory> #include <string> -#include <util/system.h> +class ArgsManager; struct NodeContext; namespace interfaces { struct BlockAndHeaderTipInfo; @@ -25,21 +25,21 @@ class Ref; void Interrupt(NodeContext& node); void Shutdown(NodeContext& node); //!Initialize the logging infrastructure -void InitLogging(); +void InitLogging(const ArgsManager& args); //!Parameter interaction: change current parameters depending on various rules -void InitParameterInteraction(); +void InitParameterInteraction(ArgsManager& args); /** Initialize bitcoin core: Basic context setup. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read. */ -bool AppInitBasicSetup(); +bool AppInitBasicSetup(ArgsManager& args); /** * Initialization: parameter interaction. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitBasicSetup should have been called. */ -bool AppInitParameterInteraction(); +bool AppInitParameterInteraction(const ArgsManager& args); /** * Initialization sanity checks: ecc init, sanity checks, dir lock. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 21400d00f8..206262eb03 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -57,27 +57,13 @@ class NodeImpl : public Node { public: NodeImpl(NodeContext* context) { setContext(context); } - void initError(const bilingual_str& message) override { InitError(message); } - bool parseParameters(int argc, const char* const argv[], std::string& error) override - { - return gArgs.ParseParameters(argc, argv, error); - } - bool readConfigFiles(std::string& error) override { return gArgs.ReadConfigFiles(error, true); } - void forceSetArg(const std::string& arg, const std::string& value) override { gArgs.ForceSetArg(arg, value); } - bool softSetArg(const std::string& arg, const std::string& value) override { return gArgs.SoftSetArg(arg, value); } - bool softSetBoolArg(const std::string& arg, bool value) override { return gArgs.SoftSetBoolArg(arg, value); } - void selectParams(const std::string& network) override { SelectParams(network); } - bool initSettings(std::string& error) override { return gArgs.InitSettings(error); } - uint64_t getAssumedBlockchainSize() override { return Params().AssumedBlockchainSize(); } - uint64_t getAssumedChainStateSize() override { return Params().AssumedChainStateSize(); } - std::string getNetwork() override { return Params().NetworkIDString(); } - void initLogging() override { InitLogging(); } - void initParameterInteraction() override { InitParameterInteraction(); } + void initLogging() override { InitLogging(*Assert(m_context->args)); } + void initParameterInteraction() override { InitParameterInteraction(*Assert(m_context->args)); } bilingual_str getWarnings() override { return GetWarnings(true); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - return AppInitBasicSetup() && AppInitParameterInteraction() && AppInitSanityChecks() && + return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs) && AppInitSanityChecks() && AppInitLockDataDirectory(); } bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override @@ -109,7 +95,6 @@ public: StopMapPort(); } } - void setupServerArgs() override { return SetupServerArgs(*m_context); } bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 753f3e6b13..0cff7ae3a1 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -55,41 +55,6 @@ class Node public: virtual ~Node() {} - //! Send init error. - virtual void initError(const bilingual_str& message) = 0; - - //! Set command line arguments. - virtual bool parseParameters(int argc, const char* const argv[], std::string& error) = 0; - - //! Set a command line argument - virtual void forceSetArg(const std::string& arg, const std::string& value) = 0; - - //! Set a command line argument if it doesn't already have a value - virtual bool softSetArg(const std::string& arg, const std::string& value) = 0; - - //! Set a command line boolean argument if it doesn't already have a value - virtual bool softSetBoolArg(const std::string& arg, bool value) = 0; - - //! Load settings from configuration file. - virtual bool readConfigFiles(std::string& error) = 0; - - //! Choose network parameters. - virtual void selectParams(const std::string& network) = 0; - - //! Read and update <datadir>/settings.json file with saved settings. This - //! needs to be called after selectParams() because the settings file - //! location is network-specific. - virtual bool initSettings(std::string& error) = 0; - - //! Get the (assumed) blockchain size. - virtual uint64_t getAssumedBlockchainSize() = 0; - - //! Get the (assumed) chain state size. - virtual uint64_t getAssumedChainStateSize() = 0; - - //! Get network name. - virtual std::string getNetwork() = 0; - //! Init logging. virtual void initLogging() = 0; @@ -117,9 +82,6 @@ public: //! Return whether shutdown was requested. virtual bool shutdownRequested() = 0; - //! Setup arguments - virtual void setupServerArgs() = 0; - //! Map port. virtual void mapPort(bool use_upnp) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 7fd24425cf..937e602fb0 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -37,6 +37,7 @@ namespace { //! Construct wallet tx struct. WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) { + LOCK(wallet.cs_wallet); WalletTx result; result.tx = wtx.tx; result.txin_is_mine.reserve(wtx.tx->vin.size()); @@ -132,7 +133,11 @@ public: { return m_wallet->SignMessage(message, pkhash, str_sig); } - bool isSpendable(const CTxDestination& dest) override { return m_wallet->IsMine(dest) & ISMINE_SPENDABLE; } + bool isSpendable(const CTxDestination& dest) override + { + LOCK(m_wallet->cs_wallet); + return m_wallet->IsMine(dest) & ISMINE_SPENDABLE; + } bool haveWatchOnly() override { auto spk_man = m_wallet->GetLegacyScriptPubKeyMan(); diff --git a/src/net.cpp b/src/net.cpp index 6c1980735c..883e57bdf0 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -530,6 +530,8 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) } X(nLastSend); X(nLastRecv); + X(nLastTXTime); + X(nLastBlockTime); X(nTimeConnected); X(nTimeOffset); stats.addrName = GetAddrName(); @@ -619,6 +619,8 @@ public: bool fRelayTxes; int64_t nLastSend; int64_t nLastRecv; + int64_t nLastTXTime; + int64_t nLastBlockTime; int64_t nTimeConnected; int64_t nTimeOffset; std::string addrName; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7b83583e41..60bdfbe9f5 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -844,7 +844,7 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName), pnode->IsInboundConn(), pnode->IsManualConn())); } if(!pnode->IsInboundConn()) - PushNodeVersion(*pnode, *connman, GetTime()); + PushNodeVersion(*pnode, m_connman, GetTime()); } void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const @@ -855,7 +855,7 @@ void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const // Sanity check: all unbroadcast txns should exist in the mempool if (m_mempool.exists(elem.first)) { LOCK(cs_main); - RelayTransaction(elem.first, elem.second, *connman); + RelayTransaction(elem.first, elem.second, m_connman); } else { m_mempool.RemoveUnbroadcastTx(elem.first, true); } @@ -1197,8 +1197,8 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) - : connman(connmanIn), +PeerLogicValidation::PeerLogicValidation(CConnman& connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) + : m_connman(connman), m_banman(banman), m_chainman(chainman), m_mempool(pool), @@ -1326,7 +1326,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } - connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { + m_connman.ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { AssertLockHeld(cs_main); // TODO: Avoid the repeated-serialization here @@ -1341,7 +1341,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", hashBlock.ToString(), pnode->GetId()); - connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); + m_connman.PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; } }); @@ -1353,7 +1353,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: */ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; - connman->SetBestHeight(nNewHeight); + m_connman.SetBestHeight(nNewHeight); SetServiceFlagsIBDCache(!fInitialDownload); if (!fInitialDownload) { @@ -1370,7 +1370,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } } // Relay inventory, but don't relay old inventory during initial block download. - connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { + m_connman.ForEachNode([nNewHeight, &vHashes](CNode* pnode) { LOCK(pnode->cs_inventory); if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { for (const uint256& hash : reverse_iterate(vHashes)) { @@ -1378,7 +1378,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } } }); - connman->WakeMessageHandler(); + m_connman.WakeMessageHandler(); } } @@ -1409,7 +1409,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidatio !::ChainstateActive().IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, m_connman); } } if (it != mapBlockSource.end()) @@ -2302,17 +2302,9 @@ static void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv, const CChainPar connman.PushMessage(&peer, std::move(msg)); } -void ProcessMessage( - CNode& pfrom, - const std::string& msg_type, - CDataStream& vRecv, - const std::chrono::microseconds time_received, - const CChainParams& chainparams, - ChainstateManager& chainman, - CTxMemPool& mempool, - CConnman& connman, - BanMan* banman, - const std::atomic<bool>& interruptMsgProc) +void PeerLogicValidation::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, + const CChainParams& chainparams, const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom.GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -2348,7 +2340,7 @@ void ProcessMessage( nServices = ServiceFlags(nServiceInt); if (!pfrom.IsInboundConn()) { - connman.SetServices(pfrom.addr, nServices); + m_connman.SetServices(pfrom.addr, nServices); } if (pfrom.ExpectServicesFromConn() && !HasAllDesirableServiceFlags(nServices)) { @@ -2377,7 +2369,7 @@ void ProcessMessage( if (!vRecv.empty()) vRecv >> fRelay; // Disconnect if we connected to ourself - if (pfrom.IsInboundConn() && !connman.CheckIncomingNonce(nNonce)) + if (pfrom.IsInboundConn() && !m_connman.CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom.addr.ToString()); pfrom.fDisconnect = true; @@ -2391,13 +2383,13 @@ void ProcessMessage( // Be shy and don't send version until we hear if (pfrom.IsInboundConn()) - PushNodeVersion(pfrom, connman, GetAdjustedTime()); + PushNodeVersion(pfrom, m_connman, GetAdjustedTime()); if (nVersion >= WTXID_RELAY_VERSION) { - connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::WTXIDRELAY)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::WTXIDRELAY)); } - connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom.nServices = nServices; pfrom.SetAddrLocal(addrMe); @@ -2453,9 +2445,9 @@ void ProcessMessage( } // Get recent addresses - connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); pfrom.fGetAddr = true; - connman.MarkAddressGood(pfrom.addr); + m_connman.MarkAddressGood(pfrom.addr); } std::string remoteAddr; @@ -2474,7 +2466,7 @@ void ProcessMessage( // If the peer is old enough to have the old alert system, send it the final alert. if (pfrom.nVersion <= 70012) { CDataStream finalAlert(ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50"), SER_NETWORK, PROTOCOL_VERSION); - connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); } // Feeler connections exist only to verify if address is online. @@ -2513,7 +2505,7 @@ void ProcessMessage( // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom.nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks @@ -2524,9 +2516,9 @@ void ProcessMessage( bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; if (pfrom.GetLocalServices() & NODE_WITNESS) - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } pfrom.fSuccessfullyConnected = true; return; @@ -2590,7 +2582,7 @@ void ProcessMessage( if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom.AddAddressKnown(addr); - if (banman && (banman->IsDiscouraged(addr) || banman->IsBanned(addr))) { + if (m_banman && (m_banman->IsDiscouraged(addr) || m_banman->IsBanned(addr))) { // Do not process banned/discouraged addresses beyond remembering we received them continue; } @@ -2598,13 +2590,13 @@ void ProcessMessage( if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, connman); + RelayAddress(addr, fReachable, m_connman); } // Do not store addresses outside our network if (fReachable) vAddrOk.push_back(addr); } - connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); + m_connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; if (pfrom.IsAddrFetchConn()) @@ -2680,7 +2672,7 @@ void ProcessMessage( if (inv.IsMsgWtx()) continue; } - bool fAlreadyHave = AlreadyHave(inv, mempool); + bool fAlreadyHave = AlreadyHave(inv, m_mempool); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); if (inv.IsMsgTx()) { @@ -2703,14 +2695,14 @@ void ProcessMessage( LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; return; - } else if (!fAlreadyHave && !chainman.ActiveChainstate().IsInitialBlockDownload()) { + } else if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { RequestTx(State(pfrom.GetId()), ToGenTxid(inv), current_time); } } } if (best_block != nullptr) { - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom.GetId()); } @@ -2734,7 +2726,7 @@ void ProcessMessage( } pfrom.vRecvGetData.insert(pfrom.vRecvGetData.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom, chainparams, connman, mempool, interruptMsgProc); + ProcessGetData(pfrom, chainparams, m_connman, m_mempool, interruptMsgProc); return; } @@ -2818,7 +2810,7 @@ void ProcessMessage( // Unlock cs_most_recent_block to avoid cs_main lock inversion } if (recent_block) { - SendBlockTransactions(*recent_block, req, pfrom, connman); + SendBlockTransactions(*recent_block, req, pfrom, m_connman); return; } @@ -2851,7 +2843,7 @@ void ProcessMessage( bool ret = ReadBlockFromDisk(block, pindex, chainparams.GetConsensus()); assert(ret); - SendBlockTransactions(block, req, pfrom, connman); + SendBlockTransactions(block, req, pfrom, m_connman); return; } @@ -2918,7 +2910,7 @@ void ProcessMessage( // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip(); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); return; } @@ -2977,10 +2969,10 @@ void ProcessMessage( // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (!AlreadyHave(CInv(MSG_WTX, wtxid), mempool) && - AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - mempool.check(&::ChainstateActive().CoinsTip()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); + if (!AlreadyHave(CInv(MSG_WTX, wtxid), m_mempool) && + AcceptToMemoryPool(m_mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + m_mempool.check(&::ChainstateActive().CoinsTip()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), m_connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(txid, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -2995,10 +2987,10 @@ void ProcessMessage( LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom.GetId(), tx.GetHash().ToString(), - mempool.size(), mempool.DynamicMemoryUsage() / 1000); + m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one - ProcessOrphanTx(connman, mempool, pfrom.orphan_work_set, lRemovedTxn); + ProcessOrphanTx(m_connman, m_mempool, pfrom.orphan_work_set, lRemovedTxn); } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { @@ -3032,7 +3024,7 @@ void ProcessMessage( // protocol for getting all unconfirmed parents. CInv _inv(MSG_TX | nFetchFlags, parent_txid); pfrom.AddKnownTx(parent_txid); - if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), ToGenTxid(_inv), current_time); + if (!AlreadyHave(_inv, m_mempool)) RequestTx(State(pfrom.GetId()), ToGenTxid(_inv), current_time); } AddOrphanTx(ptx, pfrom.GetId()); @@ -3093,11 +3085,11 @@ void ProcessMessage( // if they were already in the mempool, // allowing the node to function as a gateway for // nodes hidden behind it. - if (!mempool.exists(tx.GetHash())) { + if (!m_mempool.exists(tx.GetHash())) { LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), m_connman); } } } @@ -3150,7 +3142,7 @@ void ProcessMessage( if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return; } @@ -3161,7 +3153,7 @@ void ProcessMessage( const CBlockIndex *pindex = nullptr; BlockValidationState state; - if (!chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { + if (!m_chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return; @@ -3211,7 +3203,7 @@ void ProcessMessage( // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return; } @@ -3232,9 +3224,9 @@ void ProcessMessage( if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = nullptr; - if (!MarkBlockAsInFlight(mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(m_mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) - (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&m_mempool)); else { // The block was already in flight using compact blocks from the same peer LogPrint(BCLog::NET, "Peer sent us compact block we were already syncing!\n"); @@ -3252,7 +3244,7 @@ void ProcessMessage( // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } @@ -3269,7 +3261,7 @@ void ProcessMessage( fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -3277,7 +3269,7 @@ void ProcessMessage( // download from. // Optimistically try to reconstruct anyway since we might be // able to without any round trips. - PartiallyDownloadedBlock tempBlock(&mempool); + PartiallyDownloadedBlock tempBlock(&m_mempool); ReadStatus status = tempBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status != READ_STATUS_OK) { // TODO: don't ignore failures @@ -3295,7 +3287,7 @@ void ProcessMessage( // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -3305,7 +3297,7 @@ void ProcessMessage( } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, chainparams, chainman, mempool, connman, banman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, chainparams, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be @@ -3313,7 +3305,7 @@ void ProcessMessage( // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that // will be detected and the peer will be disconnected/discouraged. - return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); + return ProcessHeadersMessage(pfrom, m_connman, m_chainman, m_mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } if (fBlockReconstructed) { @@ -3333,7 +3325,7 @@ void ProcessMessage( // we have a chain with at least nMinimumChainWork), and we ignore // compact blocks with less work than our tip, it is safe to treat // reconstructed compact blocks as having been requested. - chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + m_chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom.nLastBlockTime = GetTime(); } else { @@ -3385,7 +3377,7 @@ void ProcessMessage( // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -3423,7 +3415,7 @@ void ProcessMessage( // disk-space attacks), but this should be safe due to the // protections in the compact block handler -- see related comment // in compact block optimistic reconstruction handling. - chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + m_chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom.nLastBlockTime = GetTime(); } else { @@ -3457,7 +3449,7 @@ void ProcessMessage( ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - return ProcessHeadersMessage(pfrom, connman, chainman, mempool, headers, chainparams, /*via_compact_block=*/false); + return ProcessHeadersMessage(pfrom, m_connman, m_chainman, m_mempool, headers, chainparams, /*via_compact_block=*/false); } if (msg_type == NetMsgType::BLOCK) @@ -3486,7 +3478,7 @@ void ProcessMessage( mapBlockSource.emplace(hash, std::make_pair(pfrom.GetId(), true)); } bool fNewBlock = false; - chainman.ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); + m_chainman.ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); if (fNewBlock) { pfrom.nLastBlockTime = GetTime(); } else { @@ -3522,9 +3514,9 @@ void ProcessMessage( pfrom.vAddrToSend.clear(); std::vector<CAddress> vAddr; if (pfrom.HasPermission(PF_ADDR)) { - vAddr = connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); + vAddr = m_connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); } else { - vAddr = connman.GetAddresses(pfrom.addr.GetNetwork(), MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); + vAddr = m_connman.GetAddresses(pfrom.addr.GetNetwork(), MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); } FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { @@ -3544,7 +3536,7 @@ void ProcessMessage( return; } - if (connman.OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) + if (m_connman.OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) { if (!pfrom.HasPermission(PF_NOBAN)) { @@ -3577,7 +3569,7 @@ void ProcessMessage( // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } return; } @@ -3717,17 +3709,17 @@ void ProcessMessage( } if (msg_type == NetMsgType::GETCFILTERS) { - ProcessGetCFilters(pfrom, vRecv, chainparams, connman); + ProcessGetCFilters(pfrom, vRecv, chainparams, m_connman); return; } if (msg_type == NetMsgType::GETCFHEADERS) { - ProcessGetCFHeaders(pfrom, vRecv, chainparams, connman); + ProcessGetCFHeaders(pfrom, vRecv, chainparams, m_connman); return; } if (msg_type == NetMsgType::GETCFCHECKPT) { - ProcessGetCFCheckPt(pfrom, vRecv, chainparams, connman); + ProcessGetCFCheckPt(pfrom, vRecv, chainparams, m_connman); return; } @@ -3802,7 +3794,7 @@ bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) // Normal case: Disconnect the peer and discourage all nodes sharing the address LogPrintf("Disconnecting and discouraging peer %d!\n", peer_id); if (m_banman) m_banman->Discourage(pnode.addr); - connman->DisconnectNode(pnode.addr); + m_connman.DisconnectNode(pnode.addr); return true; } @@ -3820,12 +3812,12 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fMoreWork = false; if (!pfrom->vRecvGetData.empty()) - ProcessGetData(*pfrom, chainparams, *connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, chainparams, m_connman, m_mempool, interruptMsgProc); if (!pfrom->orphan_work_set.empty()) { std::list<CTransactionRef> removed_txn; LOCK2(cs_main, g_cs_orphans); - ProcessOrphanTx(*connman, m_mempool, pfrom->orphan_work_set, removed_txn); + ProcessOrphanTx(m_connman, m_mempool, pfrom->orphan_work_set, removed_txn); for (const CTransactionRef& removedTx : removed_txn) { AddToCompactExtraTransactions(removedTx); } @@ -3851,7 +3843,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman->GetReceiveFloodSize(); + pfrom->fPauseRecv = pfrom->nProcessQueueSize > m_connman.GetReceiveFloodSize(); fMoreWork = !pfrom->vProcessMsg.empty(); } CNetMessage& msg(msgs.front()); @@ -3885,7 +3877,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } try { - ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, *connman, m_banman, interruptMsgProc); + ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) @@ -3938,7 +3930,7 @@ void PeerLogicValidation::ConsiderEviction(CNode& pto, int64_t time_in_seconds) } else { assert(state.m_chain_sync.m_work_header); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - connman->PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes // Bump the timeout to allow a response, which could clear the timeout @@ -3955,7 +3947,7 @@ void PeerLogicValidation::ConsiderEviction(CNode& pto, int64_t time_in_seconds) void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) { // Check whether we have too many outbound peers - int extra_peers = connman->GetExtraOutboundCount(); + int extra_peers = m_connman.GetExtraOutboundCount(); if (extra_peers > 0) { // If we have more outbound peers than we target, disconnect one. // Pick the outbound peer that least recently announced @@ -3964,7 +3956,7 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) NodeId worst_peer = -1; int64_t oldest_block_announcement = std::numeric_limits<int64_t>::max(); - connman->ForEachNode([&](CNode* pnode) { + m_connman.ForEachNode([&](CNode* pnode) { AssertLockHeld(cs_main); // Ignore non-outbound peers, or nodes marked for disconnect already @@ -3981,7 +3973,7 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) } }); if (worst_peer != -1) { - bool disconnected = connman->ForNode(worst_peer, [&](CNode *pnode) { + bool disconnected = m_connman.ForNode(worst_peer, [&](CNode *pnode) { AssertLockHeld(cs_main); // Only disconnect a peer that has been connected to us for @@ -4005,7 +3997,7 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) // detected a stale tip. Don't try any more extra peers until // we next detect a stale tip, to limit the load we put on the // network from these extra connections. - connman->SetTryNewOutboundPeer(false); + m_connman.SetTryNewOutboundPeer(false); } } } @@ -4015,8 +4007,6 @@ void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params { LOCK(cs_main); - if (connman == nullptr) return; - int64_t time_in_seconds = GetTime(); EvictExtraOutboundPeers(time_in_seconds); @@ -4024,11 +4014,11 @@ void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params if (time_in_seconds > m_stale_tip_check_time) { // Check whether our tip is stale, and if so, allow using an extra // outbound peer - if (!fImporting && !fReindex && connman->GetNetworkActive() && connman->GetUseAddrmanOutgoing() && TipMayBeStale(consensusParams)) { + if (!fImporting && !fReindex && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale(consensusParams)) { LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); - connman->SetTryNewOutboundPeer(true); - } else if (connman->GetTryNewOutboundPeer()) { - connman->SetTryNewOutboundPeer(false); + m_connman.SetTryNewOutboundPeer(true); + } else if (m_connman.GetTryNewOutboundPeer()) { + m_connman.SetTryNewOutboundPeer(false); } m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; } @@ -4091,11 +4081,11 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->m_ping_start = GetTime<std::chrono::microseconds>(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); } } @@ -4130,14 +4120,14 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // receiver rejects addr messages larger than MAX_ADDR_TO_SEND if (vAddr.size() >= MAX_ADDR_TO_SEND) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); // we only send the big addr message once if (pto->vAddrToSend.capacity() > 40) pto->vAddrToSend.shrink_to_fit(); @@ -4164,7 +4154,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (pindexStart->pprev) pindexStart = pindexStart->pprev; LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexStart), uint256())); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexStart), uint256())); } } @@ -4248,10 +4238,10 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) - connman->PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); else { CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); - connman->PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } fGotBlockFromCache = true; } @@ -4261,7 +4251,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); assert(ret); CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); - connman->PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { @@ -4274,7 +4264,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - connman->PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -4319,7 +4309,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) for (const uint256& hash : pto->vInventoryBlockToSend) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -4332,7 +4322,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; if (pto->IsInboundConn()) { - pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; + pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{m_connman.PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; } else { // Use half the delay for outbound peers, as there is less privacy concern for them. pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, std::chrono::seconds{INVENTORY_BROADCAST_INTERVAL >> 1}); @@ -4372,7 +4362,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -4448,7 +4438,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } } if (vInv.size() == MAX_INV_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } pto->m_tx_relay->filterInventoryKnown.insert(hash); @@ -4465,7 +4455,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } } if (!vInv.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling current_time = GetTime<std::chrono::microseconds>(); @@ -4593,7 +4583,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); vGetData.push_back(inv); if (vGetData.size() >= MAX_GETDATA_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); vGetData.clear(); } UpdateTxRequestTime(gtxid, current_time); @@ -4623,7 +4613,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!vGetData.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); // // Message: feefilter @@ -4651,7 +4641,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // We always have a fee filter of at least minRelayTxFee filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); if (filterToSend != pto->m_tx_relay->lastSentFeeFilter) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); pto->m_tx_relay->lastSentFeeFilter = filterToSend; } pto->m_tx_relay->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); diff --git a/src/net_processing.h b/src/net_processing.h index 2d98714122..74d6603747 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -11,6 +11,7 @@ #include <sync.h> #include <validationinterface.h> +class CChainParams; class CTxMemPool; class ChainstateManager; @@ -28,7 +29,7 @@ static const int DISCOURAGEMENT_THRESHOLD{100}; class PeerLogicValidation final : public CValidationInterface, public NetEventsInterface { private: - CConnman* const connman; + CConnman& m_connman; /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* const m_banman; ChainstateManager& m_chainman; @@ -37,7 +38,7 @@ private: bool MaybeDiscourageAndDisconnect(CNode& pnode); public: - PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); + PeerLogicValidation(CConnman& connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); /** * Overridden from CValidationInterface. @@ -85,8 +86,14 @@ public: /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ void ReattemptInitialBroadcast(CScheduler& scheduler) const; + /** Process a single message from a peer. Public for fuzz testing */ + void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, const CChainParams& chainparams, + const std::atomic<bool>& interruptMsgProc); + private: int64_t m_stale_tip_check_time; //!< Next time to check for stale tip + }; struct CNodeStateStats { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index d29aed6c8b..8adfe38dc9 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -3,79 +3,91 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <cstdint> #include <netaddress.h> + #include <hash.h> +#include <tinyformat.h> #include <util/strencodings.h> #include <util/asmap.h> -#include <tinyformat.h> -static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; -static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; +#include <algorithm> +#include <array> +#include <cstdint> +#include <iterator> +#include <tuple> -// 0xFD + sha256("bitcoin")[0:5] -static const unsigned char g_internal_prefix[] = { 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 }; +constexpr size_t CNetAddr::V1_SERIALIZATION_SIZE; /** * Construct an unspecified IPv6 network address (::/128). * * @note This address is considered invalid by CNetAddr::IsValid() */ -CNetAddr::CNetAddr() -{ - memset(ip, 0, sizeof(ip)); -} +CNetAddr::CNetAddr() {} void CNetAddr::SetIP(const CNetAddr& ipIn) { + // Size check. + switch (ipIn.m_net) { + case NET_IPV4: + assert(ipIn.m_addr.size() == ADDR_IPV4_SIZE); + break; + case NET_IPV6: + assert(ipIn.m_addr.size() == ADDR_IPV6_SIZE); + break; + case NET_ONION: + assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE); + break; + case NET_INTERNAL: + assert(ipIn.m_addr.size() == ADDR_INTERNAL_SIZE); + break; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + m_net = ipIn.m_net; - memcpy(ip, ipIn.ip, sizeof(ip)); + m_addr = ipIn.m_addr; +} + +template <typename T1, size_t PREFIX_LEN> +inline bool HasPrefix(const T1& obj, const std::array<uint8_t, PREFIX_LEN>& prefix) +{ + return obj.size() >= PREFIX_LEN && + std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); } -void CNetAddr::SetLegacyIPv6(const uint8_t ipv6[16]) +void CNetAddr::SetLegacyIPv6(Span<const uint8_t> ipv6) { - if (memcmp(ipv6, pchIPv4, sizeof(pchIPv4)) == 0) { + assert(ipv6.size() == ADDR_IPV6_SIZE); + + size_t skip{0}; + + if (HasPrefix(ipv6, IPV4_IN_IPV6_PREFIX)) { + // IPv4-in-IPv6 m_net = NET_IPV4; - } else if (memcmp(ipv6, pchOnionCat, sizeof(pchOnionCat)) == 0) { + skip = sizeof(IPV4_IN_IPV6_PREFIX); + } else if (HasPrefix(ipv6, TORV2_IN_IPV6_PREFIX)) { + // TORv2-in-IPv6 m_net = NET_ONION; - } else if (memcmp(ipv6, g_internal_prefix, sizeof(g_internal_prefix)) == 0) { + skip = sizeof(TORV2_IN_IPV6_PREFIX); + } else if (HasPrefix(ipv6, INTERNAL_IN_IPV6_PREFIX)) { + // Internal-in-IPv6 m_net = NET_INTERNAL; + skip = sizeof(INTERNAL_IN_IPV6_PREFIX); } else { + // IPv6 m_net = NET_IPV6; } - memcpy(ip, ipv6, 16); -} -void CNetAddr::SetRaw(Network network, const uint8_t *ip_in) -{ - switch(network) - { - case NET_IPV4: - m_net = NET_IPV4; - memcpy(ip, pchIPv4, 12); - memcpy(ip+12, ip_in, 4); - break; - case NET_IPV6: - SetLegacyIPv6(ip_in); - break; - default: - assert(!"invalid network"); - } + m_addr.assign(ipv6.begin() + skip, ipv6.end()); } /** - * Try to make this a dummy address that maps the specified name into IPv6 like - * so: (0xFD + %sha256("bitcoin")[0:5]) + %sha256(name)[0:10]. Such dummy - * addresses have a prefix of fd6b:88c0:8724::/48 and are guaranteed to not be - * publicly routable as it falls under RFC4193's fc00::/7 subnet allocated to - * unique-local addresses. - * - * CAddrMan uses these fake addresses to keep track of which DNS seeds were - * used. - * + * Create an "internal" address that represents a name or FQDN. CAddrMan uses + * these fake addresses to keep track of which DNS seeds were used. * @returns Whether or not the operation was successful. - * - * @see CNetAddr::IsInternal(), CNetAddr::IsRFC4193() + * @see NET_INTERNAL, INTERNAL_IN_IPV6_PREFIX, CNetAddr::IsInternal(), CNetAddr::IsRFC4193() */ bool CNetAddr::SetInternal(const std::string &name) { @@ -85,31 +97,26 @@ bool CNetAddr::SetInternal(const std::string &name) m_net = NET_INTERNAL; unsigned char hash[32] = {}; CSHA256().Write((const unsigned char*)name.data(), name.size()).Finalize(hash); - memcpy(ip, g_internal_prefix, sizeof(g_internal_prefix)); - memcpy(ip + sizeof(g_internal_prefix), hash, sizeof(ip) - sizeof(g_internal_prefix)); + m_addr.assign(hash, hash + ADDR_INTERNAL_SIZE); return true; } /** - * Try to make this a dummy address that maps the specified onion address into - * IPv6 using OnionCat's range and encoding. Such dummy addresses have a prefix - * of fd87:d87e:eb43::/48 and are guaranteed to not be publicly routable as they - * fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. + * Parse a TORv2 address and set this object to it. * * @returns Whether or not the operation was successful. * - * @see CNetAddr::IsTor(), CNetAddr::IsRFC4193() + * @see CNetAddr::IsTor() */ bool CNetAddr::SetSpecial(const std::string &strName) { if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { std::vector<unsigned char> vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); - if (vchAddr.size() != 16-sizeof(pchOnionCat)) + if (vchAddr.size() != ADDR_TORV2_SIZE) { return false; + } m_net = NET_ONION; - memcpy(ip, pchOnionCat, sizeof(pchOnionCat)); - for (unsigned int i=0; i<16-sizeof(pchOnionCat); i++) - ip[i + sizeof(pchOnionCat)] = vchAddr[i]; + m_addr.assign(vchAddr.begin(), vchAddr.end()); return true; } return false; @@ -117,28 +124,23 @@ bool CNetAddr::SetSpecial(const std::string &strName) CNetAddr::CNetAddr(const struct in_addr& ipv4Addr) { - SetRaw(NET_IPV4, (const uint8_t*)&ipv4Addr); + m_net = NET_IPV4; + const uint8_t* ptr = reinterpret_cast<const uint8_t*>(&ipv4Addr); + m_addr.assign(ptr, ptr + ADDR_IPV4_SIZE); } CNetAddr::CNetAddr(const struct in6_addr& ipv6Addr, const uint32_t scope) { - SetRaw(NET_IPV6, (const uint8_t*)&ipv6Addr); + SetLegacyIPv6(Span<const uint8_t>(reinterpret_cast<const uint8_t*>(&ipv6Addr), sizeof(ipv6Addr))); scopeId = scope; } -unsigned int CNetAddr::GetByte(int n) const -{ - return ip[15-n]; -} - bool CNetAddr::IsBindAny() const { - const int cmplen = IsIPv4() ? 4 : 16; - for (int i = 0; i < cmplen; ++i) { - if (GetByte(i)) return false; + if (!IsIPv4() && !IsIPv6()) { + return false; } - - return true; + return std::all_of(m_addr.begin(), m_addr.end(), [](uint8_t b) { return b == 0; }); } bool CNetAddr::IsIPv4() const { return m_net == NET_IPV4; } @@ -148,88 +150,88 @@ bool CNetAddr::IsIPv6() const { return m_net == NET_IPV6; } bool CNetAddr::IsRFC1918() const { return IsIPv4() && ( - GetByte(3) == 10 || - (GetByte(3) == 192 && GetByte(2) == 168) || - (GetByte(3) == 172 && (GetByte(2) >= 16 && GetByte(2) <= 31))); + m_addr[0] == 10 || + (m_addr[0] == 192 && m_addr[1] == 168) || + (m_addr[0] == 172 && m_addr[1] >= 16 && m_addr[1] <= 31)); } bool CNetAddr::IsRFC2544() const { - return IsIPv4() && GetByte(3) == 198 && (GetByte(2) == 18 || GetByte(2) == 19); + return IsIPv4() && m_addr[0] == 198 && (m_addr[1] == 18 || m_addr[1] == 19); } bool CNetAddr::IsRFC3927() const { - return IsIPv4() && (GetByte(3) == 169 && GetByte(2) == 254); + return IsIPv4() && HasPrefix(m_addr, std::array<uint8_t, 2>{169, 254}); } bool CNetAddr::IsRFC6598() const { - return IsIPv4() && GetByte(3) == 100 && GetByte(2) >= 64 && GetByte(2) <= 127; + return IsIPv4() && m_addr[0] == 100 && m_addr[1] >= 64 && m_addr[1] <= 127; } bool CNetAddr::IsRFC5737() const { - return IsIPv4() && ((GetByte(3) == 192 && GetByte(2) == 0 && GetByte(1) == 2) || - (GetByte(3) == 198 && GetByte(2) == 51 && GetByte(1) == 100) || - (GetByte(3) == 203 && GetByte(2) == 0 && GetByte(1) == 113)); + return IsIPv4() && (HasPrefix(m_addr, std::array<uint8_t, 3>{192, 0, 2}) || + HasPrefix(m_addr, std::array<uint8_t, 3>{198, 51, 100}) || + HasPrefix(m_addr, std::array<uint8_t, 3>{203, 0, 113})); } bool CNetAddr::IsRFC3849() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x0D && GetByte(12) == 0xB8; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 4>{0x20, 0x01, 0x0D, 0xB8}); } bool CNetAddr::IsRFC3964() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x02; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 2>{0x20, 0x02}); } bool CNetAddr::IsRFC6052() const { - static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; - return IsIPv6() && memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0; + return IsIPv6() && + HasPrefix(m_addr, std::array<uint8_t, 12>{0x00, 0x64, 0xFF, 0x9B, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}); } bool CNetAddr::IsRFC4380() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && - GetByte(12) == 0; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 4>{0x20, 0x01, 0x00, 0x00}); } bool CNetAddr::IsRFC4862() const { - static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; - return IsIPv6() && memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 8>{0xFE, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}); } bool CNetAddr::IsRFC4193() const { - return IsIPv6() && (GetByte(15) & 0xFE) == 0xFC; + return IsIPv6() && (m_addr[0] & 0xFE) == 0xFC; } bool CNetAddr::IsRFC6145() const { - static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; - return IsIPv6() && memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0; + return IsIPv6() && + HasPrefix(m_addr, std::array<uint8_t, 12>{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00}); } bool CNetAddr::IsRFC4843() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 3>{0x20, 0x01, 0x00}) && + (m_addr[3] & 0xF0) == 0x10; } bool CNetAddr::IsRFC7343() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 3>{0x20, 0x01, 0x00}) && + (m_addr[3] & 0xF0) == 0x20; } bool CNetAddr::IsHeNet() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70); + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 4>{0x20, 0x01, 0x04, 0x70}); } /** @@ -243,13 +245,15 @@ bool CNetAddr::IsTor() const { return m_net == NET_ONION; } bool CNetAddr::IsLocal() const { // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) - if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) + if (IsIPv4() && (m_addr[0] == 127 || m_addr[0] == 0)) { return true; + } // IPv6 loopback (::1/128) static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (IsIPv6() && memcmp(ip, pchLocal, 16) == 0) + if (IsIPv6() && memcmp(m_addr.data(), pchLocal, sizeof(pchLocal)) == 0) { return true; + } return false; } @@ -272,13 +276,16 @@ bool CNetAddr::IsValid() const // header20 vectorlen3 addr26 addr26 addr26 header20 vectorlen3 addr26 addr26 addr26... // so if the first length field is garbled, it reads the second batch // of addr misaligned by 3 bytes. - if (IsIPv6() && memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) + if (IsIPv6() && memcmp(m_addr.data(), IPV4_IN_IPV6_PREFIX.data() + 3, + sizeof(IPV4_IN_IPV6_PREFIX) - 3) == 0) { return false; + } // unspecified IPv6 address (::/128) unsigned char ipNone6[16] = {}; - if (IsIPv6() && memcmp(ip, ipNone6, 16) == 0) + if (IsIPv6() && memcmp(m_addr.data(), ipNone6, sizeof(ipNone6)) == 0) { return false; + } // documentation IPv6 address if (IsRFC3849()) @@ -287,17 +294,11 @@ bool CNetAddr::IsValid() const if (IsInternal()) return false; - if (IsIPv4()) - { - // INADDR_NONE - uint32_t ipNone = INADDR_NONE; - if (memcmp(ip+12, &ipNone, 4) == 0) - return false; - - // 0 - ipNone = 0; - if (memcmp(ip+12, &ipNone, 4) == 0) + if (IsIPv4()) { + const uint32_t addr = ReadBE32(m_addr.data()); + if (addr == INADDR_ANY || addr == INADDR_NONE) { return false; + } } return true; @@ -318,7 +319,7 @@ bool CNetAddr::IsRoutable() const } /** - * @returns Whether or not this is a dummy address that maps a name into IPv6. + * @returns Whether or not this is a dummy address that represents a name. * * @see CNetAddr::SetInternal(const std::string &) */ @@ -341,9 +342,9 @@ enum Network CNetAddr::GetNetwork() const std::string CNetAddr::ToStringIP() const { if (IsTor()) - return EncodeBase32(&ip[6], 10) + ".onion"; + return EncodeBase32(m_addr) + ".onion"; if (IsInternal()) - return EncodeBase32(ip + sizeof(g_internal_prefix), sizeof(ip) - sizeof(g_internal_prefix)) + ".internal"; + return EncodeBase32(m_addr) + ".internal"; CService serv(*this, 0); struct sockaddr_storage sockaddr; socklen_t socklen = sizeof(sockaddr); @@ -353,13 +354,13 @@ std::string CNetAddr::ToStringIP() const return std::string(name); } if (IsIPv4()) - return strprintf("%u.%u.%u.%u", GetByte(3), GetByte(2), GetByte(1), GetByte(0)); - else - return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - GetByte(15) << 8 | GetByte(14), GetByte(13) << 8 | GetByte(12), - GetByte(11) << 8 | GetByte(10), GetByte(9) << 8 | GetByte(8), - GetByte(7) << 8 | GetByte(6), GetByte(5) << 8 | GetByte(4), - GetByte(3) << 8 | GetByte(2), GetByte(1) << 8 | GetByte(0)); + return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); + assert(IsIPv6()); + return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", + m_addr[0] << 8 | m_addr[1], m_addr[2] << 8 | m_addr[3], + m_addr[4] << 8 | m_addr[5], m_addr[6] << 8 | m_addr[7], + m_addr[8] << 8 | m_addr[9], m_addr[10] << 8 | m_addr[11], + m_addr[12] << 8 | m_addr[13], m_addr[14] << 8 | m_addr[15]); } std::string CNetAddr::ToString() const @@ -369,12 +370,12 @@ std::string CNetAddr::ToString() const bool operator==(const CNetAddr& a, const CNetAddr& b) { - return a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) == 0; + return a.m_net == b.m_net && a.m_addr == b.m_addr; } bool operator<(const CNetAddr& a, const CNetAddr& b) { - return a.m_net < b.m_net || (a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) < 0); + return std::tie(a.m_net, a.m_addr) < std::tie(b.m_net, b.m_addr); } /** @@ -391,7 +392,8 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const { if (!IsIPv4()) return false; - memcpy(pipv4Addr, ip+12, 4); + assert(sizeof(*pipv4Addr) == m_addr.size()); + memcpy(pipv4Addr, m_addr.data(), m_addr.size()); return true; } @@ -410,7 +412,8 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const if (!IsIPv6()) { return false; } - memcpy(pipv6Addr, ip, 16); + assert(sizeof(*pipv6Addr) == m_addr.size()); + memcpy(pipv6Addr, m_addr.data(), m_addr.size()); return true; } @@ -421,15 +424,17 @@ bool CNetAddr::HasLinkedIPv4() const uint32_t CNetAddr::GetLinkedIPv4() const { - if (IsIPv4() || IsRFC6145() || IsRFC6052()) { - // IPv4, mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address - return ReadBE32(ip + 12); + if (IsIPv4()) { + return ReadBE32(m_addr.data()); + } else if (IsRFC6052() || IsRFC6145()) { + // mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address + return ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); } else if (IsRFC3964()) { // 6to4 tunneled IPv4: the IPv4 address is in bytes 2-6 - return ReadBE32(ip + 2); + return ReadBE32(MakeSpan(m_addr).subspan(2, ADDR_IPV4_SIZE).data()); } else if (IsRFC4380()) { // Teredo tunneled IPv4: the IPv4 address is in the last 4 bytes of the address, but bitflipped - return ~ReadBE32(ip + 12); + return ~ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); } assert(false); } @@ -458,10 +463,10 @@ uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { } std::vector<bool> ip_bits(128); if (HasLinkedIPv4()) { - // For lookup, treat as if it was just an IPv4 address (pchIPv4 prefix + IPv4 bits) + // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (pchIPv4[byte_i] >> (7 - bit_i)) & 1; + ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; } } uint32_t ipv4 = GetLinkedIPv4(); @@ -470,8 +475,9 @@ uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { } } else { // Use all 128 bits of the IPv6 address otherwise + assert(IsIPv6()); for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { - uint8_t cur_byte = GetByte(15 - byte_i); + uint8_t cur_byte = m_addr[byte_i]; for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; } @@ -507,19 +513,15 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co } vchRet.push_back(net_class); - int nStartByte = 0; - int nBits = 16; + int nBits{0}; if (IsLocal()) { // all local addresses belong to the same group - nBits = 0; } else if (IsInternal()) { // all internal-usage addresses get their own group - nStartByte = sizeof(g_internal_prefix); - nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; + nBits = ADDR_INTERNAL_SIZE * 8; } else if (!IsRoutable()) { // all other unroutable addresses belong to the same group - nBits = 0; } else if (HasLinkedIPv4()) { // IPv4 addresses (and mapped IPv4 addresses) use /16 groups uint32_t ipv4 = GetLinkedIPv4(); @@ -527,7 +529,6 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; } else if (IsTor()) { - nStartByte = 6; nBits = 4; } else if (IsHeNet()) { // for he.net, use /36 groups @@ -537,23 +538,29 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co nBits = 32; } - // push our ip onto vchRet byte by byte... - while (nBits >= 8) - { - vchRet.push_back(GetByte(15 - nStartByte)); - nStartByte++; - nBits -= 8; - } + // Push our address onto vchRet. + const size_t num_bytes = nBits / 8; + vchRet.insert(vchRet.end(), m_addr.begin(), m_addr.begin() + num_bytes); + nBits %= 8; // ...for the last byte, push nBits and for the rest of the byte push 1's - if (nBits > 0) - vchRet.push_back(GetByte(15 - nStartByte) | ((1 << (8 - nBits)) - 1)); + if (nBits > 0) { + assert(num_bytes < m_addr.size()); + vchRet.push_back(m_addr[num_bytes] | ((1 << (8 - nBits)) - 1)); + } return vchRet; } +std::vector<unsigned char> CNetAddr::GetAddrBytes() const +{ + uint8_t serialized[V1_SERIALIZATION_SIZE]; + SerializeV1Array(serialized); + return {std::begin(serialized), std::end(serialized)}; +} + uint64_t CNetAddr::GetHash() const { - uint256 hash = Hash(ip); + uint256 hash = Hash(m_addr); uint64_t nRet; memcpy(&nRet, &hash, sizeof(nRet)); return nRet; @@ -764,53 +771,89 @@ CSubNet::CSubNet(): memset(netmask, 0, sizeof(netmask)); } -CSubNet::CSubNet(const CNetAddr &addr, int32_t mask) +CSubNet::CSubNet(const CNetAddr& addr, uint8_t mask) : CSubNet() { - valid = true; + valid = (addr.IsIPv4() && mask <= ADDR_IPV4_SIZE * 8) || + (addr.IsIPv6() && mask <= ADDR_IPV6_SIZE * 8); + if (!valid) { + return; + } + + assert(mask <= sizeof(netmask) * 8); + network = addr; - // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address - memset(netmask, 255, sizeof(netmask)); - - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - const int astartofs = network.IsIPv4() ? 12 : 0; - - int32_t n = mask; - if(n >= 0 && n <= (128 - astartofs*8)) // Only valid if in range of bits of address - { - n += astartofs*8; - // Clear bits [n..127] - for (; n < 128; ++n) - netmask[n>>3] &= ~(1<<(7-(n&7))); - } else - valid = false; - // Normalize network according to netmask - for(int x=0; x<16; ++x) - network.ip[x] &= netmask[x]; + uint8_t n = mask; + for (size_t i = 0; i < network.m_addr.size(); ++i) { + const uint8_t bits = n < 8 ? n : 8; + netmask[i] = (uint8_t)((uint8_t)0xFF << (8 - bits)); // Set first bits. + network.m_addr[i] &= netmask[i]; // Normalize network according to netmask. + n -= bits; + } } -CSubNet::CSubNet(const CNetAddr &addr, const CNetAddr &mask) +/** + * @returns The number of 1-bits in the prefix of the specified subnet mask. If + * the specified subnet mask is not a valid one, -1. + */ +static inline int NetmaskBits(uint8_t x) { - valid = true; - network = addr; - // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address - memset(netmask, 255, sizeof(netmask)); + switch(x) { + case 0x00: return 0; + case 0x80: return 1; + case 0xc0: return 2; + case 0xe0: return 3; + case 0xf0: return 4; + case 0xf8: return 5; + case 0xfc: return 6; + case 0xfe: return 7; + case 0xff: return 8; + default: return -1; + } +} + +CSubNet::CSubNet(const CNetAddr& addr, const CNetAddr& mask) : CSubNet() +{ + valid = (addr.IsIPv4() || addr.IsIPv6()) && addr.m_net == mask.m_net; + if (!valid) { + return; + } + // Check if `mask` contains 1-bits after 0-bits (which is an invalid netmask). + bool zeros_found = false; + for (auto b : mask.m_addr) { + const int num_bits = NetmaskBits(b); + if (num_bits == -1 || (zeros_found && num_bits != 0)) { + valid = false; + return; + } + if (num_bits < 8) { + zeros_found = true; + } + } - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - const int astartofs = network.IsIPv4() ? 12 : 0; + assert(mask.m_addr.size() <= sizeof(netmask)); - for(int x=astartofs; x<16; ++x) - netmask[x] = mask.ip[x]; + memcpy(netmask, mask.m_addr.data(), mask.m_addr.size()); + + network = addr; // Normalize network according to netmask - for(int x=0; x<16; ++x) - network.ip[x] &= netmask[x]; + for (size_t x = 0; x < network.m_addr.size(); ++x) { + network.m_addr[x] &= netmask[x]; + } } -CSubNet::CSubNet(const CNetAddr &addr): - valid(addr.IsValid()) +CSubNet::CSubNet(const CNetAddr& addr) : CSubNet() { - memset(netmask, 255, sizeof(netmask)); + valid = addr.IsIPv4() || addr.IsIPv6(); + if (!valid) { + return; + } + + assert(addr.m_addr.size() <= sizeof(netmask)); + + memset(netmask, 0xFF, addr.m_addr.size()); + network = addr; } @@ -822,68 +865,29 @@ bool CSubNet::Match(const CNetAddr &addr) const { if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; - for(int x=0; x<16; ++x) - if ((addr.ip[x] & netmask[x]) != network.ip[x]) + assert(network.m_addr.size() == addr.m_addr.size()); + for (size_t x = 0; x < addr.m_addr.size(); ++x) { + if ((addr.m_addr[x] & netmask[x]) != network.m_addr[x]) { return false; - return true; -} - -/** - * @returns The number of 1-bits in the prefix of the specified subnet mask. If - * the specified subnet mask is not a valid one, -1. - */ -static inline int NetmaskBits(uint8_t x) -{ - switch(x) { - case 0x00: return 0; - case 0x80: return 1; - case 0xc0: return 2; - case 0xe0: return 3; - case 0xf0: return 4; - case 0xf8: return 5; - case 0xfc: return 6; - case 0xfe: return 7; - case 0xff: return 8; - default: return -1; + } } + return true; } std::string CSubNet::ToString() const { - /* Parse binary 1{n}0{N-n} to see if mask can be represented as /n */ - int cidr = 0; - bool valid_cidr = true; - int n = network.IsIPv4() ? 12 : 0; - for (; n < 16 && netmask[n] == 0xff; ++n) - cidr += 8; - if (n < 16) { - int bits = NetmaskBits(netmask[n]); - if (bits < 0) - valid_cidr = false; - else - cidr += bits; - ++n; - } - for (; n < 16 && valid_cidr; ++n) - if (netmask[n] != 0x00) - valid_cidr = false; - - /* Format output */ - std::string strNetmask; - if (valid_cidr) { - strNetmask = strprintf("%u", cidr); - } else { - if (network.IsIPv4()) - strNetmask = strprintf("%u.%u.%u.%u", netmask[12], netmask[13], netmask[14], netmask[15]); - else - strNetmask = strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - netmask[0] << 8 | netmask[1], netmask[2] << 8 | netmask[3], - netmask[4] << 8 | netmask[5], netmask[6] << 8 | netmask[7], - netmask[8] << 8 | netmask[9], netmask[10] << 8 | netmask[11], - netmask[12] << 8 | netmask[13], netmask[14] << 8 | netmask[15]); + assert(network.m_addr.size() <= sizeof(netmask)); + + uint8_t cidr = 0; + + for (size_t i = 0; i < network.m_addr.size(); ++i) { + if (netmask[i] == 0x00) { + break; + } + cidr += NetmaskBits(netmask[i]); } - return network.ToString() + "/" + strNetmask; + return network.ToString() + strprintf("/%u", cidr); } bool CSubNet::IsValid() const diff --git a/src/netaddress.h b/src/netaddress.h index 0365907d44..d00f5a6f55 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -9,9 +9,12 @@ #include <config/bitcoin-config.h> #endif +#include <attributes.h> #include <compat.h> +#include <prevector.h> #include <serialize.h> +#include <array> #include <cstdint> #include <string> #include <vector> @@ -39,16 +42,49 @@ enum Network /// TORv2 NET_ONION, - /// A set of dummy addresses that map a name to an IPv6 address. These - /// addresses belong to RFC4193's fc00::/7 subnet (unique-local addresses). - /// We use them to map a string or FQDN to an IPv6 address in CAddrMan to - /// keep track of which DNS seeds were used. + /// A set of addresses that represent the hash of a string or FQDN. We use + /// them in CAddrMan to keep track of which DNS seeds were used. NET_INTERNAL, /// Dummy value to indicate the number of NET_* constants. NET_MAX, }; +/// Prefix of an IPv6 address when it contains an embedded IPv4 address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +static const std::array<uint8_t, 12> IPV4_IN_IPV6_PREFIX{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF +}; + +/// Prefix of an IPv6 address when it contains an embedded TORv2 address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +/// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they +/// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. +static const std::array<uint8_t, 6> TORV2_IN_IPV6_PREFIX{ + 0xFD, 0x87, 0xD8, 0x7E, 0xEB, 0x43 +}; + +/// Prefix of an IPv6 address when it contains an embedded "internal" address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +/// The prefix comes from 0xFD + SHA256("bitcoin")[0:5]. +/// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they +/// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. +static const std::array<uint8_t, 6> INTERNAL_IN_IPV6_PREFIX{ + 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 // 0xFD + sha256("bitcoin")[0:5]. +}; + +/// Size of IPv4 address (in bytes). +static constexpr size_t ADDR_IPV4_SIZE = 4; + +/// Size of IPv6 address (in bytes). +static constexpr size_t ADDR_IPV6_SIZE = 16; + +/// Size of TORv2 address (in bytes). +static constexpr size_t ADDR_TORV2_SIZE = 10; + +/// Size of "internal" (NET_INTERNAL) address (in bytes). +static constexpr size_t ADDR_INTERNAL_SIZE = 10; + /** * Network address. */ @@ -56,11 +92,16 @@ class CNetAddr { protected: /** + * Raw representation of the network address. + * In network byte order (big endian) for IPv4 and IPv6. + */ + prevector<ADDR_IPV6_SIZE, uint8_t> m_addr{ADDR_IPV6_SIZE, 0x0}; + + /** * Network to which this address belongs. */ Network m_net{NET_IPV6}; - unsigned char ip[16]; // in network byte order uint32_t scopeId{0}; // for scoped/link-local ipv6 addresses public: @@ -74,13 +115,7 @@ class CNetAddr * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy * `addr` encoding. */ - void SetLegacyIPv6(const uint8_t ipv6[16]); - - /** - * Set raw IPv4 or IPv6 address (in network byte order) - * @note Only NET_IPV4 and NET_IPV6 are allowed for network. - */ - void SetRaw(Network network, const uint8_t *data); + void SetLegacyIPv6(Span<const uint8_t> ipv6); bool SetInternal(const std::string& name); @@ -111,7 +146,6 @@ class CNetAddr enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; - unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; uint32_t GetNetClass() const; @@ -127,7 +161,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector<bool> &asmap) const; std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; - std::vector<unsigned char> GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } + std::vector<unsigned char> GetAddrBytes() const; int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); @@ -143,7 +177,7 @@ class CNetAddr template <typename Stream> void Serialize(Stream& s) const { - s << ip; + SerializeV1Stream(s); } /** @@ -152,14 +186,92 @@ class CNetAddr template <typename Stream> void Unserialize(Stream& s) { - unsigned char ip_temp[sizeof(ip)]; - s >> ip_temp; + UnserializeV1Stream(s); + } + + friend class CSubNet; + + private: + /** + * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). + */ + static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; + + /** + * Serialize in pre-ADDRv2/BIP155 format to an array. + * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. + */ + void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const + { + size_t prefix_size; + + switch (m_net) { + case NET_IPV6: + assert(m_addr.size() == sizeof(arr)); + memcpy(arr, m_addr.data(), m_addr.size()); + return; + case NET_IPV4: + prefix_size = sizeof(IPV4_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, IPV4_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_ONION: + prefix_size = sizeof(TORV2_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, TORV2_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_INTERNAL: + prefix_size = sizeof(INTERNAL_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); + } + + /** + * Serialize in pre-ADDRv2/BIP155 format to a stream. + * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. + */ + template <typename Stream> + void SerializeV1Stream(Stream& s) const + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + + SerializeV1Array(serialized); + + s << serialized; + } + + /** + * Unserialize from a pre-ADDRv2/BIP155 format from an array. + */ + void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) + { // Use SetLegacyIPv6() so that m_net is set correctly. For example // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). - SetLegacyIPv6(ip_temp); + SetLegacyIPv6(arr); } - friend class CSubNet; + /** + * Unserialize from a pre-ADDRv2/BIP155 format from a stream. + */ + template <typename Stream> + void UnserializeV1Stream(Stream& s) + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + + s >> serialized; + + UnserializeV1Array(serialized); + } }; class CSubNet @@ -174,11 +286,11 @@ class CSubNet public: CSubNet(); - CSubNet(const CNetAddr &addr, int32_t mask); - CSubNet(const CNetAddr &addr, const CNetAddr &mask); + CSubNet(const CNetAddr& addr, uint8_t mask); + CSubNet(const CNetAddr& addr, const CNetAddr& mask); //constructor for single ip subnet (<ipv4>/32 or <ipv6>/128) - explicit CSubNet(const CNetAddr &addr); + explicit CSubNet(const CNetAddr& addr); bool Match(const CNetAddr &addr) const; diff --git a/src/netbase.cpp b/src/netbase.cpp index 3a3b5f3e66..0273839017 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -13,6 +13,7 @@ #include <atomic> #include <cstdint> +#include <limits> #ifndef WIN32 #include <fcntl.h> @@ -838,8 +839,8 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) if (slash != strSubnet.npos) { std::string strNetmask = strSubnet.substr(slash + 1); - int32_t n; - if (ParseInt32(strNetmask, &n)) { + uint8_t n; + if (ParseUInt8(strNetmask, &n)) { // If valid number, assume CIDR variable-length subnet masking ret = CSubNet(network, n); return ret.IsValid(); diff --git a/src/protocol.h b/src/protocol.h index 9565f5ff96..2e6c767cdd 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -398,7 +398,9 @@ enum GetDataMsg : uint32_t { MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 MSG_WITNESS_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG, //!< Defined in BIP144 MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG, //!< Defined in BIP144 - MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, + // MSG_FILTERED_WITNESS_BLOCK is defined in BIP144 as reserved for future + // use and remains unused. + // MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, }; /** inv message data */ diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index f53fcc41f3..e63ffdfb36 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -27,9 +27,11 @@ #include <qt/walletmodel.h> #endif // ENABLE_WALLET +#include <init.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <node/context.h> +#include <node/ui_interface.h> #include <noui.h> #include <uint256.h> #include <util/system.h> @@ -37,6 +39,7 @@ #include <util/translation.h> #include <validation.h> +#include <boost/signals2/connection.hpp> #include <memory> #include <QApplication> @@ -193,10 +196,9 @@ void BitcoinCore::shutdown() static int qt_argc = 1; static const char* qt_argv = "bitcoin-qt"; -BitcoinApplication::BitcoinApplication(interfaces::Node& node): +BitcoinApplication::BitcoinApplication(): QApplication(qt_argc, const_cast<char **>(&qt_argv)), coreThread(nullptr), - m_node(node), optionsModel(nullptr), clientModel(nullptr), window(nullptr), @@ -246,12 +248,12 @@ void BitcoinApplication::createPaymentServer() void BitcoinApplication::createOptionsModel(bool resetSettings) { - optionsModel = new OptionsModel(m_node, this, resetSettings); + optionsModel = new OptionsModel(this, resetSettings); } void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) { - window = new BitcoinGUI(m_node, platformStyle, networkStyle, nullptr); + window = new BitcoinGUI(node(), platformStyle, networkStyle, nullptr); pollShutdownTimer = new QTimer(window); connect(pollShutdownTimer, &QTimer::timeout, window, &BitcoinGUI::detectShutdown); @@ -259,17 +261,26 @@ void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) { - SplashScreen *splash = new SplashScreen(m_node, nullptr, networkStyle); + assert(!m_splash); + m_splash = new SplashScreen(nullptr, networkStyle); // We don't hold a direct pointer to the splash screen after creation, but the splash // screen will take care of deleting itself when finish() happens. - splash->show(); - connect(this, &BitcoinApplication::splashFinished, splash, &SplashScreen::finish); - connect(this, &BitcoinApplication::requestedShutdown, splash, &QWidget::close); + m_splash->show(); + connect(this, &BitcoinApplication::splashFinished, m_splash, &SplashScreen::finish); + connect(this, &BitcoinApplication::requestedShutdown, m_splash, &QWidget::close); +} + +void BitcoinApplication::setNode(interfaces::Node& node) +{ + assert(!m_node); + m_node = &node; + if (optionsModel) optionsModel->setNode(*m_node); + if (m_splash) m_splash->setNode(*m_node); } bool BitcoinApplication::baseInitialize() { - return m_node.baseInitialize(); + return node().baseInitialize(); } void BitcoinApplication::startThread() @@ -277,7 +288,7 @@ void BitcoinApplication::startThread() if(coreThread) return; coreThread = new QThread(this); - BitcoinCore *executor = new BitcoinCore(m_node); + BitcoinCore *executor = new BitcoinCore(node()); executor->moveToThread(coreThread); /* communication to and from thread */ @@ -298,8 +309,8 @@ void BitcoinApplication::parameterSetup() // print to the console unnecessarily. gArgs.SoftSetBoolArg("-printtoconsole", false); - m_node.initLogging(); - m_node.initParameterInteraction(); + InitLogging(gArgs); + InitParameterInteraction(gArgs); } void BitcoinApplication::InitializePruneSetting(bool prune) @@ -331,7 +342,7 @@ void BitcoinApplication::requestShutdown() window->unsubscribeFromCoreSignals(); // Request node shutdown, which can interrupt long operations, like // rescanning a wallet. - m_node.startShutdown(); + node().startShutdown(); // Unsetting the client model can cause the current thread to wait for node // to complete an operation, like wait for a RPC execution to complete. window->setClientModel(nullptr); @@ -353,7 +364,7 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead { // Log this only after AppInitMain finishes, as then logging setup is guaranteed complete qInfo() << "Platform customization:" << platformStyle->getName(); - clientModel = new ClientModel(m_node, optionsModel); + clientModel = new ClientModel(node(), optionsModel); window->setClientModel(clientModel, &tip_info); #ifdef ENABLE_WALLET if (WalletModel::isWalletEnabled()) { @@ -437,9 +448,9 @@ int GuiMain(int argc, char* argv[]) std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(&node_context); // Subscribe to global signals from core - std::unique_ptr<interfaces::Handler> handler_message_box = node->handleMessageBox(noui_ThreadSafeMessageBox); - std::unique_ptr<interfaces::Handler> handler_question = node->handleQuestion(noui_ThreadSafeQuestion); - std::unique_ptr<interfaces::Handler> handler_init_message = node->handleInitMessage(noui_InitMessage); + boost::signals2::scoped_connection handler_message_box = ::uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBox); + boost::signals2::scoped_connection handler_question = ::uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestion); + boost::signals2::scoped_connection handler_init_message = ::uiInterface.InitMessage_connect(noui_InitMessage); // Do not refer to data directory yet, this can be overridden by Intro::pickDataDirectory @@ -453,15 +464,15 @@ int GuiMain(int argc, char* argv[]) QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling); #endif - BitcoinApplication app(*node); + BitcoinApplication app; /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: - node->setupServerArgs(); + SetupServerArgs(node_context); SetupUIArgs(gArgs); std::string error; - if (!node->parseParameters(argc, argv, error)) { - node->initError(strprintf(Untranslated("Error parsing command line arguments: %s\n"), error)); + if (!gArgs.ParseParameters(argc, argv, error)) { + InitError(strprintf(Untranslated("Error parsing command line arguments: %s\n"), error)); // Create a message box, because the gui has neither been created nor has subscribed to core signals QMessageBox::critical(nullptr, PACKAGE_NAME, // message can not be translated because translations have not been initialized @@ -487,7 +498,7 @@ int GuiMain(int argc, char* argv[]) // Show help message immediately after parsing command-line options (for "-lang") and setting locale, // but before showing splash screen. if (HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { - HelpMessageDialog help(*node, nullptr, gArgs.IsArgSet("-version")); + HelpMessageDialog help(nullptr, gArgs.IsArgSet("-version")); help.showOrPrint(); return EXIT_SUCCESS; } @@ -497,18 +508,18 @@ int GuiMain(int argc, char* argv[]) bool did_show_intro = false; bool prune = false; // Intro dialog prune check box // Gracefully exit if the user cancels - if (!Intro::showIfNeeded(*node, did_show_intro, prune)) return EXIT_SUCCESS; + if (!Intro::showIfNeeded(did_show_intro, prune)) return EXIT_SUCCESS; /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes if (!CheckDataDirOption()) { - node->initError(strprintf(Untranslated("Specified data directory \"%s\" does not exist.\n"), gArgs.GetArg("-datadir", ""))); + InitError(strprintf(Untranslated("Specified data directory \"%s\" does not exist.\n"), gArgs.GetArg("-datadir", ""))); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: Specified data directory \"%1\" does not exist.").arg(QString::fromStdString(gArgs.GetArg("-datadir", "")))); return EXIT_FAILURE; } - if (!node->readConfigFiles(error)) { - node->initError(strprintf(Untranslated("Error reading configuration file: %s\n"), error)); + if (!gArgs.ReadConfigFiles(error, true)) { + InitError(strprintf(Untranslated("Error reading configuration file: %s\n"), error)); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: Cannot parse configuration file: %1.").arg(QString::fromStdString(error))); return EXIT_FAILURE; @@ -522,18 +533,18 @@ int GuiMain(int argc, char* argv[]) // Check for -chain, -testnet or -regtest parameter (Params() calls are only valid after this clause) try { - node->selectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainName()); } catch(std::exception &e) { - node->initError(Untranslated(strprintf("%s\n", e.what()))); + InitError(Untranslated(strprintf("%s\n", e.what()))); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: %1").arg(e.what())); return EXIT_FAILURE; } #ifdef ENABLE_WALLET // Parse URIs on command line -- this can affect Params() - PaymentServer::ipcParseCommandLine(*node, argc, argv); + PaymentServer::ipcParseCommandLine(argc, argv); #endif - if (!node->initSettings(error)) { - node->initError(Untranslated(error)); + if (!gArgs.InitSettings(error)) { + InitError(Untranslated(error)); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error initializing settings: %1").arg(QString::fromStdString(error))); return EXIT_FAILURE; } @@ -587,6 +598,8 @@ int GuiMain(int argc, char* argv[]) if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) app.createSplashScreen(networkStyle.data()); + app.setNode(*node); + int rv = EXIT_SUCCESS; try { @@ -609,10 +622,10 @@ int GuiMain(int argc, char* argv[]) } } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(node->getWarnings().translated)); + app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } catch (...) { PrintExceptionContinue(nullptr, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(node->getWarnings().translated)); + app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } return rv; } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 20c6dfc047..69e0a5921e 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -10,6 +10,7 @@ #endif #include <QApplication> +#include <assert.h> #include <memory> #include <interfaces/node.h> @@ -20,6 +21,7 @@ class NetworkStyle; class OptionsModel; class PaymentServer; class PlatformStyle; +class SplashScreen; class WalletController; class WalletModel; @@ -54,7 +56,7 @@ class BitcoinApplication: public QApplication { Q_OBJECT public: - explicit BitcoinApplication(interfaces::Node& node); + explicit BitcoinApplication(); ~BitcoinApplication(); #ifdef ENABLE_WALLET @@ -88,6 +90,9 @@ public: /// Setup platform style void setupPlatformStyle(); + interfaces::Node& node() const { assert(m_node); return *m_node; } + void setNode(interfaces::Node& node); + public Q_SLOTS: void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); void shutdownResult(); @@ -102,7 +107,6 @@ Q_SIGNALS: private: QThread *coreThread; - interfaces::Node& m_node; OptionsModel *optionsModel; ClientModel *clientModel; BitcoinGUI *window; @@ -114,6 +118,8 @@ private: int returnValue; const PlatformStyle *platformStyle; std::unique_ptr<QWidget> shutdownWindow; + SplashScreen* m_splash = nullptr; + interfaces::Node* m_node = nullptr; void startThread(); }; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 56adbf249a..8935ff19bf 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -95,7 +95,7 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty updateWindowTitle(); rpcConsole = new RPCConsole(node, _platformStyle, nullptr); - helpMessageDialog = new HelpMessageDialog(node, this, false); + helpMessageDialog = new HelpMessageDialog(this, false); #ifdef ENABLE_WALLET if(enableWallet) { @@ -821,7 +821,7 @@ void BitcoinGUI::aboutClicked() if(!clientModel) return; - HelpMessageDialog dlg(m_node, this, true); + HelpMessageDialog dlg(this, true); dlg.exec(); } diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 93840b4169..d210faec03 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -290,7 +290,7 @@ <item row="11" column="0"> <widget class="QLabel" name="label_3"> <property name="text"> - <string>Current number of blocks</string> + <string>Current block height</string> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 7f439fa45e..bab17562a6 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -21,11 +21,6 @@ #include <util/system.h> #ifdef WIN32 -#ifdef _WIN32_IE -#undef _WIN32_IE -#endif -#define _WIN32_IE 0x0501 -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif @@ -94,7 +89,7 @@ static std::string DummyAddress(const CChainParams ¶ms) std::vector<unsigned char> sourcedata = params.Base58Prefix(CChainParams::PUBKEY_ADDRESS); sourcedata.insert(sourcedata.end(), dummydata, dummydata + sizeof(dummydata)); for(int i=0; i<256; ++i) { // Try every trailing byte - std::string s = EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()); + std::string s = EncodeBase58(sourcedata); if (!IsValidDestinationString(s)) { return s; } diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index ad21dfc3ef..235722d091 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -6,6 +6,7 @@ #include <config/bitcoin-config.h> #endif +#include <chainparams.h> #include <fs.h> #include <qt/intro.h> #include <qt/forms/ui_intro.h> @@ -181,7 +182,7 @@ void Intro::setDataDirectory(const QString &dataDir) } } -bool Intro::showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& prune) +bool Intro::showIfNeeded(bool& did_show_intro, bool& prune) { did_show_intro = false; @@ -199,13 +200,13 @@ bool Intro::showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& pru { /* Use selectParams here to guarantee Params() can be used by node interface */ try { - node.selectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainName()); } catch (const std::exception&) { return false; } /* If current default data directory does not exist, let the user choose one */ - Intro intro(0, node.getAssumedBlockchainSize(), node.getAssumedChainStateSize()); + Intro intro(0, Params().AssumedBlockchainSize(), Params().AssumedChainStateSize()); intro.setDataDirectory(dataDir); intro.setWindowIcon(QIcon(":icons/bitcoin")); did_show_intro = true; @@ -242,7 +243,7 @@ bool Intro::showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& pru * (to be consistent with bitcoind behavior) */ if(dataDir != GUIUtil::getDefaultDataDirectory()) { - node.softSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting + gArgs.SoftSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting } return true; } diff --git a/src/qt/intro.h b/src/qt/intro.h index 732393246e..51f42de7ac 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -47,7 +47,7 @@ public: * @note do NOT call global GetDataDir() before calling this function, this * will cause the wrong path to be cached. */ - static bool showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& prune); + static bool showIfNeeded(bool& did_show_intro, bool& prune); Q_SIGNALS: void requestCheck(); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 58a7591c95..7e089b4f95 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -27,8 +27,8 @@ const char *DEFAULT_GUI_PROXY_HOST = "127.0.0.1"; static const QString GetDefaultProxyAddress(); -OptionsModel::OptionsModel(interfaces::Node& node, QObject *parent, bool resetSettings) : - QAbstractListModel(parent), m_node(node) +OptionsModel::OptionsModel(QObject *parent, bool resetSettings) : + QAbstractListModel(parent) { Init(resetSettings); } @@ -97,12 +97,12 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("nDatabaseCache")) settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache); - if (!m_node.softSetArg("-dbcache", settings.value("nDatabaseCache").toString().toStdString())) + if (!gArgs.SoftSetArg("-dbcache", settings.value("nDatabaseCache").toString().toStdString())) addOverriddenOption("-dbcache"); if (!settings.contains("nThreadsScriptVerif")) settings.setValue("nThreadsScriptVerif", DEFAULT_SCRIPTCHECK_THREADS); - if (!m_node.softSetArg("-par", settings.value("nThreadsScriptVerif").toString().toStdString())) + if (!gArgs.SoftSetArg("-par", settings.value("nThreadsScriptVerif").toString().toStdString())) addOverriddenOption("-par"); if (!settings.contains("strDataDir")) @@ -112,19 +112,19 @@ void OptionsModel::Init(bool resetSettings) #ifdef ENABLE_WALLET if (!settings.contains("bSpendZeroConfChange")) settings.setValue("bSpendZeroConfChange", true); - if (!m_node.softSetBoolArg("-spendzeroconfchange", settings.value("bSpendZeroConfChange").toBool())) + if (!gArgs.SoftSetBoolArg("-spendzeroconfchange", settings.value("bSpendZeroConfChange").toBool())) addOverriddenOption("-spendzeroconfchange"); #endif // Network if (!settings.contains("fUseUPnP")) settings.setValue("fUseUPnP", DEFAULT_UPNP); - if (!m_node.softSetBoolArg("-upnp", settings.value("fUseUPnP").toBool())) + if (!gArgs.SoftSetBoolArg("-upnp", settings.value("fUseUPnP").toBool())) addOverriddenOption("-upnp"); if (!settings.contains("fListen")) settings.setValue("fListen", DEFAULT_LISTEN); - if (!m_node.softSetBoolArg("-listen", settings.value("fListen").toBool())) + if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) addOverriddenOption("-listen"); if (!settings.contains("fUseProxy")) @@ -132,7 +132,7 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("addrProxy")) settings.setValue("addrProxy", GetDefaultProxyAddress()); // Only try to set -proxy, if user has enabled fUseProxy - if (settings.value("fUseProxy").toBool() && !m_node.softSetArg("-proxy", settings.value("addrProxy").toString().toStdString())) + if ((settings.value("fUseProxy").toBool() && !gArgs.SoftSetArg("-proxy", settings.value("addrProxy").toString().toStdString()))) addOverriddenOption("-proxy"); else if(!settings.value("fUseProxy").toBool() && !gArgs.GetArg("-proxy", "").empty()) addOverriddenOption("-proxy"); @@ -142,7 +142,7 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("addrSeparateProxyTor")) settings.setValue("addrSeparateProxyTor", GetDefaultProxyAddress()); // Only try to set -onion, if user has enabled fUseSeparateProxyTor - if (settings.value("fUseSeparateProxyTor").toBool() && !m_node.softSetArg("-onion", settings.value("addrSeparateProxyTor").toString().toStdString())) + if ((settings.value("fUseSeparateProxyTor").toBool() && !gArgs.SoftSetArg("-onion", settings.value("addrSeparateProxyTor").toString().toStdString()))) addOverriddenOption("-onion"); else if(!settings.value("fUseSeparateProxyTor").toBool() && !gArgs.GetArg("-onion", "").empty()) addOverriddenOption("-onion"); @@ -150,7 +150,7 @@ void OptionsModel::Init(bool resetSettings) // Display if (!settings.contains("language")) settings.setValue("language", ""); - if (!m_node.softSetArg("-lang", settings.value("language").toString().toStdString())) + if (!gArgs.SoftSetArg("-lang", settings.value("language").toString().toStdString())) addOverriddenOption("-lang"); language = settings.value("language").toString(); @@ -244,10 +244,10 @@ void OptionsModel::SetPruneEnabled(bool prune, bool force) const int64_t prune_target_mib = PruneGBtoMiB(settings.value("nPruneSize").toInt()); std::string prune_val = prune ? ToString(prune_target_mib) : "0"; if (force) { - m_node.forceSetArg("-prune", prune_val); + gArgs.ForceSetArg("-prune", prune_val); return; } - if (!m_node.softSetArg("-prune", prune_val)) { + if (!gArgs.SoftSetArg("-prune", prune_val)) { addOverriddenOption("-prune"); } } @@ -353,7 +353,7 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in break; case MapPortUPnP: // core option - can be changed on-the-fly settings.setValue("fUseUPnP", value.toBool()); - m_node.mapPort(value.toBool()); + node().mapPort(value.toBool()); break; case MinimizeOnClose: fMinimizeOnClose = value.toBool(); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 14fdf9046e..3d9e7bbb80 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -11,6 +11,8 @@ #include <QAbstractListModel> +#include <assert.h> + namespace interfaces { class Node; } @@ -39,7 +41,7 @@ class OptionsModel : public QAbstractListModel Q_OBJECT public: - explicit OptionsModel(interfaces::Node& node, QObject *parent = nullptr, bool resetSettings = false); + explicit OptionsModel(QObject *parent = nullptr, bool resetSettings = false); enum OptionID { StartAtStartup, // bool @@ -92,10 +94,11 @@ public: void setRestartRequired(bool fRequired); bool isRestartRequired() const; - interfaces::Node& node() const { return m_node; } + interfaces::Node& node() const { assert(m_node); return *m_node; } + void setNode(interfaces::Node& node) { assert(!m_node); m_node = &node; } private: - interfaces::Node& m_node; + interfaces::Node* m_node = nullptr; /* Qt-only settings */ bool fHideTrayIcon; bool fMinimizeToTray; diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index a1da85bda7..8679ced685 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -74,7 +74,7 @@ static QSet<QString> savedPaymentRequests; // Warning: ipcSendCommandLine() is called early in init, // so don't use "Q_EMIT message()", but "QMessageBox::"! // -void PaymentServer::ipcParseCommandLine(interfaces::Node& node, int argc, char* argv[]) +void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) { for (int i = 1; i < argc; i++) { @@ -97,11 +97,11 @@ void PaymentServer::ipcParseCommandLine(interfaces::Node& node, int argc, char* auto tempChainParams = CreateChainParams(CBaseChainParams::MAIN); if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - node.selectParams(CBaseChainParams::MAIN); + SelectParams(CBaseChainParams::MAIN); } else { tempChainParams = CreateChainParams(CBaseChainParams::TESTNET); if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - node.selectParams(CBaseChainParams::TESTNET); + SelectParams(CBaseChainParams::TESTNET); } } } diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 154f4a7ea6..eaf2bafe59 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -61,7 +61,7 @@ class PaymentServer : public QObject public: // Parse URIs on command line // Returns false on error - static void ipcParseCommandLine(interfaces::Node& node, int argc, char *argv[]); + static void ipcParseCommandLine(int argc, char *argv[]); // Returns true if there were URIs on the command line // which were successfully sent to an already-running diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 6e6b2b8466..bd63d6e7fb 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -24,8 +24,8 @@ #include <QScreen> -SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const NetworkStyle *networkStyle) : - QWidget(nullptr, f), curAlignment(0), m_node(node) +SplashScreen::SplashScreen(Qt::WindowFlags f, const NetworkStyle *networkStyle) : + QWidget(nullptr, f), curAlignment(0) { // set reference point, paddings int paddingRight = 50; @@ -124,7 +124,6 @@ SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const Netw setFixedSize(r.size()); move(QGuiApplication::primaryScreen()->geometry().center() - r.center()); - subscribeToCoreSignals(); installEventFilter(this); GUIUtil::handleCloseWindowShortcut(this); @@ -132,14 +131,28 @@ SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const Netw SplashScreen::~SplashScreen() { - unsubscribeFromCoreSignals(); + if (m_node) unsubscribeFromCoreSignals(); +} + +void SplashScreen::setNode(interfaces::Node& node) +{ + assert(!m_node); + m_node = &node; + subscribeToCoreSignals(); + if (m_shutdown) m_node->startShutdown(); +} + +void SplashScreen::shutdown() +{ + m_shutdown = true; + if (m_node) m_node->startShutdown(); } bool SplashScreen::eventFilter(QObject * obj, QEvent * ev) { if (ev->type() == QEvent::KeyPress) { QKeyEvent *keyEvent = static_cast<QKeyEvent *>(ev); if (keyEvent->key() == Qt::Key_Q) { - m_node.startShutdown(); + shutdown(); } } return QObject::eventFilter(obj, ev); @@ -183,10 +196,10 @@ void SplashScreen::ConnectWallet(std::unique_ptr<interfaces::Wallet> wallet) void SplashScreen::subscribeToCoreSignals() { // Connect signals to client - m_handler_init_message = m_node.handleInitMessage(std::bind(InitMessage, this, std::placeholders::_1)); - m_handler_show_progress = m_node.handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + m_handler_init_message = m_node->handleInitMessage(std::bind(InitMessage, this, std::placeholders::_1)); + m_handler_show_progress = m_node->handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); #ifdef ENABLE_WALLET - m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { ConnectWallet(std::move(wallet)); }); + m_handler_load_wallet = m_node->handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { ConnectWallet(std::move(wallet)); }); #endif } @@ -221,6 +234,6 @@ void SplashScreen::paintEvent(QPaintEvent *event) void SplashScreen::closeEvent(QCloseEvent *event) { - m_node.startShutdown(); // allows an "emergency" shutdown during startup + shutdown(); // allows an "emergency" shutdown during startup event->ignore(); } diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index 3158524117..2213b02c55 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -28,8 +28,9 @@ class SplashScreen : public QWidget Q_OBJECT public: - explicit SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const NetworkStyle *networkStyle); + explicit SplashScreen(Qt::WindowFlags f, const NetworkStyle *networkStyle); ~SplashScreen(); + void setNode(interfaces::Node& node); protected: void paintEvent(QPaintEvent *event) override; @@ -50,6 +51,8 @@ private: void subscribeToCoreSignals(); /** Disconnect core signals to splash screen */ void unsubscribeFromCoreSignals(); + /** Initiate shutdown */ + void shutdown(); /** Connect wallet signals to splash screen */ void ConnectWallet(std::unique_ptr<interfaces::Wallet> wallet); @@ -58,7 +61,8 @@ private: QColor curColor; int curAlignment; - interfaces::Node& m_node; + interfaces::Node* m_node = nullptr; + bool m_shutdown = false; std::unique_ptr<interfaces::Handler> m_handler_init_message; std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_load_wallet; diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 035c8196bc..84f981dff3 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -108,7 +108,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); - OptionsModel optionsModel(node); + OptionsModel optionsModel; ClientModel clientModel(node, &optionsModel); AddWallet(wallet); WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 031913bd02..86356b43c8 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -68,11 +68,11 @@ int main(int argc, char* argv[]) // Don't remove this, it's needed to access // QApplication:: and QCoreApplication:: in the tests - BitcoinApplication app(*node); + BitcoinApplication app; + app.setNode(*node); app.setApplicationName("Bitcoin-Qt-test"); - node->setupServerArgs(); // Make gArgs available in the NodeContext - node->context()->args->ClearArgs(); // Clear added args again + app.node().context()->args = &gArgs; // Make gArgs available in the NodeContext AppTests app_tests(app); if (QTest::qExec(&app_tests) != 0) { fInvalid = true; @@ -81,7 +81,7 @@ int main(int argc, char* argv[]) if (QTest::qExec(&test1) != 0) { fInvalid = true; } - RPCNestedTests test3(*node); + RPCNestedTests test3(app.node()); if (QTest::qExec(&test3) != 0) { fInvalid = true; } @@ -90,11 +90,11 @@ int main(int argc, char* argv[]) fInvalid = true; } #ifdef ENABLE_WALLET - WalletTests test5(*node); + WalletTests test5(app.node()); if (QTest::qExec(&test5) != 0) { fInvalid = true; } - AddressBookTests test6(*node); + AddressBookTests test6(app.node()); if (QTest::qExec(&test6) != 0) { fInvalid = true; } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 475fd589af..adcfe0d25c 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -163,7 +163,7 @@ void TestGUI(interfaces::Node& node) std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); - OptionsModel optionsModel(node); + OptionsModel optionsModel; ClientModel clientModel(node, &optionsModel); AddWallet(wallet); WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 01922cf996..b7f85446f4 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -28,7 +28,7 @@ #include <QVBoxLayout> /** "Help message" or "About" dialog box */ -HelpMessageDialog::HelpMessageDialog(interfaces::Node& node, QWidget *parent, bool about) : +HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : QDialog(parent), ui(new Ui::HelpMessageDialog) { diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 425b468f40..d2a5d5f67f 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -12,10 +12,6 @@ QT_BEGIN_NAMESPACE class QMainWindow; QT_END_NAMESPACE -namespace interfaces { - class Node; -} - namespace Ui { class HelpMessageDialog; } @@ -26,7 +22,7 @@ class HelpMessageDialog : public QDialog Q_OBJECT public: - explicit HelpMessageDialog(interfaces::Node& node, QWidget *parent, bool about); + explicit HelpMessageDialog(QWidget *parent, bool about); ~HelpMessageDialog(); void printToConsole(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 868ff88d08..033e00daf5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -556,7 +556,10 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) RPCResult::Type::ARR, "", "", {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}}, RPCResult{"for verbose = true", - RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, }, RPCExamples{ HelpExampleCli("getmempoolancestors", "\"mytxid\"") @@ -588,7 +591,6 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) for (CTxMemPool::txiter ancestorIt : setAncestors) { o.push_back(ancestorIt->GetTx().GetHash().ToString()); } - return o; } else { UniValue o(UniValue::VOBJ); @@ -1739,7 +1741,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) {RPCResult::Type::NUM, "total_size", "Total size of all non-coinbase transactions"}, {RPCResult::Type::NUM, "total_weight", "Total weight of all non-coinbase transactions divided by segwit scale factor (4)"}, {RPCResult::Type::NUM, "totalfee", "The fee total"}, - {RPCResult::Type::NUM, "txs", "The number of transactions (excluding coinbase)"}, + {RPCResult::Type::NUM, "txs", "The number of transactions (including coinbase)"}, {RPCResult::Type::NUM, "utxo_increase", "The increase/decrease in the number of unspent outputs"}, {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 76aa9dbfc1..d9cc6f8832 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -511,12 +511,13 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "A list of strings", { - {"support", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasetxn', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, + {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, }, }, {"rules", RPCArg::Type::ARR, RPCArg::Optional::NO, "A list of strings", { - {"support", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported softfork deployment"}, + {"segwit", RPCArg::Type::STR, RPCArg::Optional::NO, "(literal) indicates client side segwit support"}, + {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "other client side supported softfork deployment"}, }, }, }, @@ -528,7 +529,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {RPCResult::Type::NUM, "version", "The preferred block version"}, {RPCResult::Type::ARR, "rules", "specific block rules that are to be enforced", { - {RPCResult::Type::STR, "", "rulename"}, + {RPCResult::Type::STR, "", "name of a rule the client must understand to some extent; see BIP 9 for format"}, }}, {RPCResult::Type::OBJ_DYN, "vbavailable", "set of pending, supported versionbit (BIP 9) softfork deployments", { @@ -536,7 +537,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) }}, {RPCResult::Type::NUM, "vbrequired", "bit mask of versionbits the server requires set in submissions"}, {RPCResult::Type::STR, "previousblockhash", "The hash of current highest block"}, - {RPCResult::Type::ARR, "", "contents of non-coinbase transactions that should be included in the next block", + {RPCResult::Type::ARR, "transactions", "contents of non-coinbase transactions that should be included in the next block", { {RPCResult::Type::OBJ, "", "", { @@ -552,15 +553,12 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {RPCResult::Type::NUM, "weight", "total transaction weight, as counted for purposes of block limits"}, }}, }}, - {RPCResult::Type::OBJ, "coinbaseaux", "data that should be included in the coinbase's scriptSig content", + {RPCResult::Type::OBJ_DYN, "coinbaseaux", "data that should be included in the coinbase's scriptSig content", { - {RPCResult::Type::ELISION, "", ""}, + {RPCResult::Type::STR_HEX, "key", "values must be in the coinbase (keys may be ignored)"}, }}, {RPCResult::Type::NUM, "coinbasevalue", "maximum allowable input to coinbase transaction, including the generation award and transaction fees (in satoshis)"}, - {RPCResult::Type::OBJ, "coinbasetxn", "information for coinbase transaction", - { - {RPCResult::Type::ELISION, "", ""}, - }}, + {RPCResult::Type::STR, "longpollid", "an id to include with a request to longpoll on an update to this template"}, {RPCResult::Type::STR, "target", "The hash target"}, {RPCResult::Type::NUM_TIME, "mintime", "The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::ARR, "mutable", "list of ways the block template may be changed", @@ -574,6 +572,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME}, {RPCResult::Type::STR, "bits", "compressed target of next block"}, {RPCResult::Type::NUM, "height", "The height of the next block"}, + {RPCResult::Type::STR, "default_witness_commitment", /* optional */ true, "a valid witness commitment for the unmodified block template"} }}, RPCExamples{ HelpExampleCli("getblocktemplate", "'{\"rules\": [\"segwit\"]}'") diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index ff31bee1e3..0c982317f5 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,6 +4,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <httpserver.h> +#include <index/blockfilterindex.h> +#include <index/txindex.h> #include <interfaces/chain.h> #include <key_io.h> #include <node/context.h> @@ -636,6 +638,60 @@ static RPCHelpMan echo(const std::string& name) static RPCHelpMan echo() { return echo("echo"); } static RPCHelpMan echojson() { return echo("echojson"); } +static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_name) +{ + UniValue ret_summary(UniValue::VOBJ); + if (!index_name.empty() && index_name != summary.name) return ret_summary; + + UniValue entry(UniValue::VOBJ); + entry.pushKV("synced", summary.synced); + entry.pushKV("best_block_height", summary.best_block_height); + ret_summary.pushKV(summary.name, entry); + return ret_summary; +} + +static RPCHelpMan getindexinfo() +{ + return RPCHelpMan{"getindexinfo", + "\nReturns the status of one or all available indices currently running in the node.\n", + { + {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", { + { + RPCResult::Type::OBJ, "name", "The name of the index", + { + {RPCResult::Type::BOOL, "synced", "Whether the index is synced or not"}, + {RPCResult::Type::NUM, "best_block_height", "The block height to which the index is synced"}, + } + }, + }, + }, + RPCExamples{ + HelpExampleCli("getindexinfo", "") + + HelpExampleRpc("getindexinfo", "") + + HelpExampleCli("getindexinfo", "txindex") + + HelpExampleRpc("getindexinfo", "txindex") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue result(UniValue::VOBJ); + const std::string index_name = request.params[0].isNull() ? "" : request.params[0].get_str(); + + if (g_txindex) { + result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); + } + + ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { + result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); + }); + + return result; +}, + }; +} + void RegisterMiscRPCCommands(CRPCTable &t) { // clang-format off @@ -650,6 +706,7 @@ static const CRPCCommand commands[] = { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, + { "util", "getindexinfo", &getindexinfo, {"index_name"} }, /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 9bd7c15992..e9343b3348 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -100,6 +100,8 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) {RPCResult::Type::BOOL, "relaytxes", "Whether peer has asked us to relay transactions to it"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, + {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, + {RPCResult::Type::NUM_TIME, "last_block", "The " + UNIX_EPOCH_TIME + " of the last block received from this peer"}, {RPCResult::Type::NUM, "bytessent", "The total bytes sent"}, {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, @@ -169,6 +171,8 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("relaytxes", stats.fRelayTxes); obj.pushKV("lastsend", stats.nLastSend); obj.pushKV("lastrecv", stats.nLastRecv); + obj.pushKV("last_transaction", stats.nLastTXTime); + obj.pushKV("last_block", stats.nLastBlockTime); obj.pushKV("bytessent", stats.nSendBytes); obj.pushKV("bytesrecv", stats.nRecvBytes); obj.pushKV("conntime", stats.nTimeConnected); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index abc8168c55..d6988ee3ac 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1300,7 +1300,7 @@ UniValue combinepsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue finalizepsbt(const JSONRPCRequest& request) @@ -1435,7 +1435,7 @@ UniValue createpsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue converttopsbt(const JSONRPCRequest& request) @@ -1502,7 +1502,7 @@ UniValue converttopsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue utxoupdatepsbt(const JSONRPCRequest& request) @@ -1590,7 +1590,7 @@ UniValue utxoupdatepsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue joinpsbts(const JSONRPCRequest& request) @@ -1683,7 +1683,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << shuffled_psbt; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue analyzepsbt(const JSONRPCRequest& request) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 39feb4ccc9..7b2457a5e3 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1258,34 +1258,37 @@ public: } }; +/** Compute the (single) SHA256 of the concatenation of all prevouts of a tx. */ template <class T> -uint256 GetPrevoutHash(const T& txTo) +uint256 GetPrevoutsSHA256(const T& txTo) { CHashWriter ss(SER_GETHASH, 0); for (const auto& txin : txTo.vin) { ss << txin.prevout; } - return ss.GetHash(); + return ss.GetSHA256(); } +/** Compute the (single) SHA256 of the concatenation of all nSequences of a tx. */ template <class T> -uint256 GetSequenceHash(const T& txTo) +uint256 GetSequencesSHA256(const T& txTo) { CHashWriter ss(SER_GETHASH, 0); for (const auto& txin : txTo.vin) { ss << txin.nSequence; } - return ss.GetHash(); + return ss.GetSHA256(); } +/** Compute the (single) SHA256 of the concatenation of all txouts of a tx. */ template <class T> -uint256 GetOutputsHash(const T& txTo) +uint256 GetOutputsSHA256(const T& txTo) { CHashWriter ss(SER_GETHASH, 0); for (const auto& txout : txTo.vout) { ss << txout; } - return ss.GetHash(); + return ss.GetSHA256(); } } // namespace @@ -1297,9 +1300,9 @@ void PrecomputedTransactionData::Init(const T& txTo) // Cache is calculated only for transactions with witness if (txTo.HasWitness()) { - hashPrevouts = GetPrevoutHash(txTo); - hashSequence = GetSequenceHash(txTo); - hashOutputs = GetOutputsHash(txTo); + hashPrevouts = SHA256Uint256(GetPrevoutsSHA256(txTo)); + hashSequence = SHA256Uint256(GetSequencesSHA256(txTo)); + hashOutputs = SHA256Uint256(GetOutputsSHA256(txTo)); } m_ready = true; @@ -1329,16 +1332,16 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn const bool cacheready = cache && cache->m_ready; if (!(nHashType & SIGHASH_ANYONECANPAY)) { - hashPrevouts = cacheready ? cache->hashPrevouts : GetPrevoutHash(txTo); + hashPrevouts = cacheready ? cache->hashPrevouts : SHA256Uint256(GetPrevoutsSHA256(txTo)); } if (!(nHashType & SIGHASH_ANYONECANPAY) && (nHashType & 0x1f) != SIGHASH_SINGLE && (nHashType & 0x1f) != SIGHASH_NONE) { - hashSequence = cacheready ? cache->hashSequence : GetSequenceHash(txTo); + hashSequence = cacheready ? cache->hashSequence : SHA256Uint256(GetSequencesSHA256(txTo)); } if ((nHashType & 0x1f) != SIGHASH_SINGLE && (nHashType & 0x1f) != SIGHASH_NONE) { - hashOutputs = cacheready ? cache->hashOutputs : GetOutputsHash(txTo); + hashOutputs = cacheready ? cache->hashOutputs : SHA256Uint256(GetOutputsSHA256(txTo)); } else if ((nHashType & 0x1f) == SIGHASH_SINGLE && nIn < txTo.vout.size()) { CHashWriter ss(SER_GETHASH, 0); ss << txTo.vout[nIn]; diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index b4f392116c..26de780f29 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -10,7 +10,6 @@ #endif #ifdef WIN32 -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 57559fa687..6a636f2574 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58) std::vector<unsigned char> sourcedata = ParseHex(test[0].get_str()); std::string base58string = test[1].get_str(); BOOST_CHECK_MESSAGE( - EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()) == base58string, + EncodeBase58(sourcedata) == base58string, strTest); } } diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 0115803e58..c0a2fca9ca 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -80,7 +80,7 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(*connman, nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -150,7 +150,7 @@ static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidat BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { auto connman = MakeUnique<CConnmanTest>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(*connman, nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); const Consensus::Params& consensusParams = Params().GetConsensus(); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(*connman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -279,7 +279,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(*connman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); int64_t nStartTime = GetTime(); diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index 40ca01bd9f..e3aefa18a3 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -33,7 +33,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) if (buffer.size() < 1 + 3 + 4) return; int asmap_size = 3 + (buffer[0] & 127); bool ipv6 = buffer[0] & 128; - int addr_size = ipv6 ? 16 : 4; + const size_t addr_size = ipv6 ? ADDR_IPV6_SIZE : ADDR_IPV4_SIZE; if (buffer.size() < size_t(1 + asmap_size + addr_size)) return; std::vector<bool> asmap = ipv6 ? IPV6_PREFIX_ASMAP : IPV4_PREFIX_ASMAP; asmap.reserve(asmap.size() + 8 * asmap_size); @@ -43,7 +43,17 @@ void test_one_input(const std::vector<uint8_t>& buffer) } } if (!SanityCheckASMap(asmap)) return; + + const uint8_t* addr_data = buffer.data() + 1 + asmap_size; CNetAddr net_addr; - net_addr.SetRaw(ipv6 ? NET_IPV6 : NET_IPV4, buffer.data() + 1 + asmap_size); + if (ipv6) { + assert(addr_size == ADDR_IPV6_SIZE); + net_addr.SetLegacyIPv6(Span<const uint8_t>(addr_data, addr_size)); + } else { + assert(addr_size == ADDR_IPV4_SIZE); + in_addr ipv4; + memcpy(&ipv4, addr_data, addr_size); + net_addr.SetIP(CNetAddr{ipv4}); + } (void)net_addr.GetMappedAS(asmap); } diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 3597f51e51..2b181c6da1 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -52,7 +52,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); const int64_t atoi64_without_locale = atoi64(random_string); const int atoi_without_locale = atoi(random_string); - const int64_t atoi64c_without_locale = atoi64(random_string.c_str()); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to @@ -80,8 +79,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) } const int64_t atoi64_with_locale = atoi64(random_string); assert(atoi64_without_locale == atoi64_with_locale); - const int64_t atoi64c_with_locale = atoi64(random_string.c_str()); - assert(atoi64c_without_locale == atoi64c_with_locale); const int atoi_with_locale = atoi(random_string); assert(atoi_without_locale == atoi_with_locale); const std::string tostring_with_locale = ToString(random_int64); diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp new file mode 100644 index 0000000000..1ff9d6b286 --- /dev/null +++ b/src/test/fuzz/net.cpp @@ -0,0 +1,156 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <chainparamsbase.h> +#include <net.h> +#include <net_permissions.h> +#include <netaddress.h> +#include <optional.h> +#include <protocol.h> +#include <random.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> + +#include <cstdint> +#include <string> +#include <vector> + +void initialize() +{ + static const BasicTestingSetup basic_testing_setup; +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const std::optional<CAddress> address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!address) { + return; + } + const std::optional<CAddress> address_bind = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!address_bind) { + return; + } + + CNode node{fuzzed_data_provider.ConsumeIntegral<NodeId>(), + static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>()), + fuzzed_data_provider.ConsumeIntegral<int>(), + INVALID_SOCKET, + *address, + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + *address_bind, + fuzzed_data_provider.ConsumeRandomLengthString(32), + fuzzed_data_provider.PickValueInArray({ConnectionType::INBOUND, ConnectionType::OUTBOUND, ConnectionType::MANUAL, ConnectionType::FEELER, ConnectionType::BLOCK_RELAY, ConnectionType::ADDR_FETCH})}; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 12)) { + case 0: { + node.CloseSocketDisconnect(); + break; + } + case 1: { + node.MaybeSetAddrName(fuzzed_data_provider.ConsumeRandomLengthString(32)); + break; + } + case 2: { + node.SetSendVersion(fuzzed_data_provider.ConsumeIntegral<int>()); + break; + } + case 3: { + const std::vector<bool> asmap = ConsumeRandomLengthIntegralVector<bool>(fuzzed_data_provider, 128); + if (!SanityCheckASMap(asmap)) { + break; + } + CNodeStats stats; + node.copyStats(stats, asmap); + break; + } + case 4: { + node.SetRecvVersion(fuzzed_data_provider.ConsumeIntegral<int>()); + break; + } + case 5: { + const CNode* add_ref_node = node.AddRef(); + assert(add_ref_node == &node); + break; + } + case 6: { + if (node.GetRefCount() > 0) { + node.Release(); + } + break; + } + case 7: { + if (node.m_addr_known == nullptr) { + break; + } + const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!addr_opt) { + break; + } + node.AddAddressKnown(*addr_opt); + break; + } + case 8: { + if (node.m_addr_known == nullptr) { + break; + } + const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!addr_opt) { + break; + } + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + node.PushAddress(*addr_opt, fast_random_context); + break; + } + case 9: { + const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); + if (!inv_opt) { + break; + } + node.AddKnownTx(inv_opt->hash); + break; + } + case 10: { + node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); + break; + } + case 11: { + const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (!service_opt) { + break; + } + node.SetAddrLocal(*service_opt); + break; + } + case 12: { + const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); + bool complete; + node.ReceiveMsgBytes((const char*)b.data(), b.size(), complete); + break; + } + } + } + + (void)node.GetAddrLocal(); + (void)node.GetAddrName(); + (void)node.GetId(); + (void)node.GetLocalNonce(); + (void)node.GetLocalServices(); + (void)node.GetMyStartingHeight(); + (void)node.GetRecvVersion(); + const int ref_count = node.GetRefCount(); + assert(ref_count >= 0); + (void)node.GetSendVersion(); + (void)node.IsAddrRelayPeer(); + + const NetPermissionFlags net_permission_flags = fuzzed_data_provider.ConsumeBool() ? + fuzzed_data_provider.PickValueInArray<NetPermissionFlags>({NetPermissionFlags::PF_NONE, NetPermissionFlags::PF_BLOOMFILTER, NetPermissionFlags::PF_RELAY, NetPermissionFlags::PF_FORCERELAY, NetPermissionFlags::PF_NOBAN, NetPermissionFlags::PF_MEMPOOL, NetPermissionFlags::PF_ISIMPLICIT, NetPermissionFlags::PF_ALL}) : + static_cast<NetPermissionFlags>(fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + (void)node.HasPermission(net_permission_flags); +} diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 2901c704f6..8252f38726 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -17,9 +17,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CNetAddr net_addr = ConsumeNetAddr(fuzzed_data_provider); - for (int i = 0; i < 15; ++i) { - (void)net_addr.GetByte(i); - } (void)net_addr.GetHash(); (void)net_addr.GetNetClass(); if (net_addr.GetNetwork() == Network::NET_IPV4) { @@ -78,7 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)net_addr.ToString(); (void)net_addr.ToStringIP(); - const CSubNet sub_net{net_addr, fuzzed_data_provider.ConsumeIntegral<int32_t>()}; + const CSubNet sub_net{net_addr, fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; (void)sub_net.IsValid(); (void)sub_net.ToString(); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 677b87a47a..ec09acc6c6 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -30,18 +30,6 @@ #include <string> #include <vector> -void ProcessMessage( - CNode& pfrom, - const std::string& msg_type, - CDataStream& vRecv, - const std::chrono::microseconds time_received, - const CChainParams& chainparams, - ChainstateManager& chainman, - CTxMemPool& mempool, - CConnman& connman, - BanMan* banman, - const std::atomic<bool>& interruptMsgProc); - namespace { #ifdef MESSAGE_TYPE @@ -87,10 +75,9 @@ void test_one_input(const std::vector<uint8_t>& buffer) connman.AddTestNode(p2p_node); g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { - ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTime<std::chrono::microseconds>(), - Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, - *g_setup->m_node.connman, g_setup->m_node.banman.get(), - std::atomic<bool>{false}); + g_setup->m_node.peer_logic->ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, + GetTime<std::chrono::microseconds>(), Params(), + std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 9f9552edb9..ed6093a8a8 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -257,7 +257,7 @@ CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept { - return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int32_t>()}; + return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; } void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 317000c771..917ae571f5 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -13,8 +13,10 @@ #include <streams.h> #include <test/util/setup_common.h> #include <util/memory.h> +#include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <version.h> #include <boost/test/unit_test.hpp> @@ -188,6 +190,78 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode2->IsInboundConn() == true); } +BOOST_AUTO_TEST_CASE(cnetaddr_basic) +{ + CNetAddr addr; + + // IPv4, INADDR_ANY + BOOST_REQUIRE(LookupHost("0.0.0.0", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "0.0.0.0"); + + // IPv4, INADDR_NONE + BOOST_REQUIRE(LookupHost("255.255.255.255", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "255.255.255.255"); + + // IPv4, casual + BOOST_REQUIRE(LookupHost("12.34.56.78", addr, false)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "12.34.56.78"); + + // IPv6, in6addr_any + BOOST_REQUIRE(LookupHost("::", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv6()); + + BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "::"); + + // IPv6, casual + BOOST_REQUIRE(LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", addr, false)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv6()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "1122:3344:5566:7788:9900:aabb:ccdd:eeff"); + + // TORv2 + addr.SetSpecial("6hzph5hv6337r6p2.onion"); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsTor()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); + + // Internal + addr.SetInternal("esffpp"); + BOOST_REQUIRE(!addr.IsValid()); // "internal" is considered invalid + BOOST_REQUIRE(addr.IsInternal()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "esffpvrt3wpeaygy.internal"); +} + +BOOST_AUTO_TEST_CASE(cnetaddr_serialize) +{ + CNetAddr addr; + CDataStream s(SER_NETWORK, PROTOCOL_VERSION); + + addr.SetInternal("a"); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2"); + s.clear(); +} + // prior to PR #14728, this test triggers an undefined behavior BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 49073ea657..6681c92bb5 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -185,6 +185,7 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(!ResolveSubNet("1.2.3.0/-1").IsValid()); BOOST_CHECK(ResolveSubNet("1.2.3.0/32").IsValid()); BOOST_CHECK(!ResolveSubNet("1.2.3.0/33").IsValid()); + BOOST_CHECK(!ResolveSubNet("1.2.3.0/300").IsValid()); BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/0").IsValid()); BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/33").IsValid()); BOOST_CHECK(!ResolveSubNet("1:2:3:4:5:6:7:8/-1").IsValid()); @@ -216,6 +217,11 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).Match(ResolveIP("1:2:3:4:5:6:7:8"))); BOOST_CHECK(!CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).Match(ResolveIP("1:2:3:4:5:6:7:9"))); BOOST_CHECK(CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).ToString() == "1:2:3:4:5:6:7:8/128"); + // IPv4 address with IPv6 netmask or the other way around. + BOOST_CHECK(!CSubNet(ResolveIP("1.1.1.1"), ResolveIP("ffff::")).IsValid()); + BOOST_CHECK(!CSubNet(ResolveIP("::1"), ResolveIP("255.0.0.0")).IsValid()); + // Can't subnet TOR (or any other non-IPv4 and non-IPv6 network). + BOOST_CHECK(!CSubNet(ResolveIP("5wyqrzbvrdsumnok.onion"), ResolveIP("255.0.0.0")).IsValid()); subnet = ResolveSubNet("1.2.3.4/255.255.255.255"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); @@ -290,11 +296,13 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK_EQUAL(subnet.ToString(), "1::/16"); subnet = ResolveSubNet("1:2:3:4:5:6:7:8/0000:0000:0000:0000:0000:0000:0000:0000"); BOOST_CHECK_EQUAL(subnet.ToString(), "::/0"); + // Invalid netmasks (with 1-bits after 0-bits) subnet = ResolveSubNet("1.2.3.4/255.255.232.0"); - BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/255.255.232.0"); + BOOST_CHECK(!subnet.IsValid()); + subnet = ResolveSubNet("1.2.3.4/255.0.255.255"); + BOOST_CHECK(!subnet.IsValid()); subnet = ResolveSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); - BOOST_CHECK_EQUAL(subnet.ToString(), "1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); - + BOOST_CHECK(!subnet.IsValid()); } BOOST_AUTO_TEST_CASE(netbase_getgroup) @@ -428,7 +436,8 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); - BOOST_CHECK(LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); + // We only do subnetting for IPv4 and IPv6 + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index b2ae1cb845..d9a00c2205 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -97,8 +97,8 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve SelectParams(chainName); SeedInsecureRand(); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); - InitLogging(); - AppInitParameterInteraction(); + InitLogging(*m_node.args); + AppInitParameterInteraction(*m_node.args); LogInstance().StartLogging(); SHA256AutoDetect(); ECC_Start(); @@ -168,7 +168,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const m_node.mempool->setSanityCheck(1.0); m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. - m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + m_node.peer_logic = MakeUnique<PeerLogicValidation>(*m_node.connman, m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); { CConnman::Options options; options.m_msgproc = m_node.peer_logic.get(); @@ -196,49 +196,34 @@ TestingSetup::~TestingSetup() TestChain100Setup::TestChain100Setup() { - // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. - // TODO: fix the code to support SegWit blocks. - gArgs.ForceSetArg("-segwitheight", "432"); - // Need to recreate chainparams - SelectParams(CBaseChainParams::REGTEST); - // Generate a 100-block chain: coinbaseKey.MakeNewKey(true); - CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - for (int i = 0; i < COINBASE_MATURITY; i++) - { + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + for (int i = 0; i < COINBASE_MATURITY; i++) { std::vector<CMutableTransaction> noTxns; CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); m_coinbase_txns.push_back(b.vtx[0]); } } -// Create a new block with just given transactions, coinbase paying to -// scriptPubKey, and try to add it to the current chain. CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); - CBlock& block = pblocktemplate->block; + CTxMemPool empty_pool; + CBlock block = BlockAssembler(empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; - // Replace mempool-selected txns with just coinbase plus passed-in txns: - block.vtx.resize(1); - for (const CMutableTransaction& tx : txns) + Assert(block.vtx.size() == 1); + for (const CMutableTransaction& tx : txns) { block.vtx.push_back(MakeTransactionRef(tx)); - // IncrementExtraNonce creates a valid coinbase and merkleRoot - { - LOCK(cs_main); - unsigned int extraNonce = 0; - IncrementExtraNonce(&block, ::ChainActive().Tip(), extraNonce); } + RegenerateCommitments(block); while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); - CBlock result = block; - return result; + return block; } TestChain100Setup::~TestChain100Setup() @@ -246,8 +231,8 @@ TestChain100Setup::~TestChain100Setup() gArgs.ForceSetArg("-segwitheight", "0"); } - -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx) { +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) +{ return FromTx(MakeTransactionRef(tx)); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 78b279e42a..22f5d6d936 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -102,15 +102,16 @@ class CBlock; struct CMutableTransaction; class CScript; -// -// Testing fixture that pre-creates a -// 100-block REGTEST-mode block chain -// +/** + * Testing fixture that pre-creates a 100-block REGTEST-mode block chain + */ struct TestChain100Setup : public RegTestingSetup { TestChain100Setup(); - // Create a new block with just given transactions, coinbase paying to - // scriptPubKey, and try to add it to the current chain. + /** + * Create a new block with just given transactions, coinbase paying to + * scriptPubKey, and try to add it to the current chain. + */ CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey); diff --git a/src/threadsafety.h b/src/threadsafety.h index 5f2c40bac6..52bf83b676 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -18,9 +18,7 @@ #define LOCKABLE __attribute__((lockable)) #define SCOPED_LOCKABLE __attribute__((scoped_lockable)) #define GUARDED_BY(x) __attribute__((guarded_by(x))) -#define GUARDED_VAR __attribute__((guarded_var)) #define PT_GUARDED_BY(x) __attribute__((pt_guarded_by(x))) -#define PT_GUARDED_VAR __attribute__((pt_guarded_var)) #define ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__))) #define ACQUIRED_BEFORE(...) __attribute__((acquired_before(__VA_ARGS__))) #define EXCLUSIVE_LOCK_FUNCTION(...) __attribute__((exclusive_lock_function(__VA_ARGS__))) @@ -33,14 +31,12 @@ #define EXCLUSIVE_LOCKS_REQUIRED(...) __attribute__((exclusive_locks_required(__VA_ARGS__))) #define SHARED_LOCKS_REQUIRED(...) __attribute__((shared_locks_required(__VA_ARGS__))) #define NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis)) -#define ASSERT_EXCLUSIVE_LOCK(...) __attribute((assert_exclusive_lock(__VA_ARGS__))) +#define ASSERT_EXCLUSIVE_LOCK(...) __attribute__((assert_exclusive_lock(__VA_ARGS__))) #else #define LOCKABLE #define SCOPED_LOCKABLE #define GUARDED_BY(x) -#define GUARDED_VAR #define PT_GUARDED_BY(x) -#define PT_GUARDED_VAR #define ACQUIRED_AFTER(...) #define ACQUIRED_BEFORE(...) #define EXCLUSIVE_LOCK_FUNCTION(...) diff --git a/src/util/message.cpp b/src/util/message.cpp index 1e7128d225..e1d5cff48c 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -64,7 +64,7 @@ bool MessageSign( return false; } - signature = EncodeBase64(signature_bytes.data(), signature_bytes.size()); + signature = EncodeBase64(signature_bytes); return true; } diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index d10f92ffe6..079a4529a3 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -126,20 +126,20 @@ void SplitHostPort(std::string in, int &portOut, std::string &hostOut) { hostOut = in; } -std::string EncodeBase64(const unsigned char* pch, size_t len) +std::string EncodeBase64(Span<const unsigned char> input) { static const char *pbase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; std::string str; - str.reserve(((len + 2) / 3) * 4); - ConvertBits<8, 6, true>([&](int v) { str += pbase64[v]; }, pch, pch + len); + str.reserve(((input.size() + 2) / 3) * 4); + ConvertBits<8, 6, true>([&](int v) { str += pbase64[v]; }, input.begin(), input.end()); while (str.size() % 4) str += '='; return str; } std::string EncodeBase64(const std::string& str) { - return EncodeBase64((const unsigned char*)str.data(), str.size()); + return EncodeBase64(MakeUCharSpan(str)); } std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) @@ -201,20 +201,20 @@ std::string DecodeBase64(const std::string& str, bool* pf_invalid) return std::string((const char*)vchRet.data(), vchRet.size()); } -std::string EncodeBase32(const unsigned char* pch, size_t len) +std::string EncodeBase32(Span<const unsigned char> input) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; std::string str; - str.reserve(((len + 4) / 5) * 8); - ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, pch, pch + len); + str.reserve(((input.size() + 4) / 5) * 8); + ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, input.begin(), input.end()); while (str.size() % 8) str += '='; return str; } std::string EncodeBase32(const std::string& str) { - return EncodeBase32((const unsigned char*)str.data(), str.size()); + return EncodeBase32(MakeUCharSpan(str)); } std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) @@ -318,6 +318,18 @@ bool ParseInt64(const std::string& str, int64_t *out) n <= std::numeric_limits<int64_t>::max(); } +bool ParseUInt8(const std::string& str, uint8_t *out) +{ + uint32_t u32; + if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits<uint8_t>::max()) { + return false; + } + if (out != nullptr) { + *out = static_cast<uint8_t>(u32); + } + return true; +} + bool ParseUInt32(const std::string& str, uint32_t *out) { if (!ParsePrechecks(str)) @@ -407,15 +419,6 @@ std::string FormatParagraph(const std::string& in, size_t width, size_t indent) return out.str(); } -int64_t atoi64(const char* psz) -{ -#ifdef _MSC_VER - return _atoi64(psz); -#else - return strtoll(psz, nullptr, 10); -#endif -} - int64_t atoi64(const std::string& str) { #ifdef _MSC_VER diff --git a/src/util/strencodings.h b/src/util/strencodings.h index eaa0fa9992..1519214140 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -48,15 +48,14 @@ bool IsHex(const std::string& str); bool IsHexNumber(const std::string& str); std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid = nullptr); std::string DecodeBase64(const std::string& str, bool* pf_invalid = nullptr); -std::string EncodeBase64(const unsigned char* pch, size_t len); +std::string EncodeBase64(Span<const unsigned char> input); std::string EncodeBase64(const std::string& str); std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid = nullptr); std::string DecodeBase32(const std::string& str, bool* pf_invalid = nullptr); -std::string EncodeBase32(const unsigned char* pch, size_t len); +std::string EncodeBase32(Span<const unsigned char> input); std::string EncodeBase32(const std::string& str); void SplitHostPort(std::string in, int& portOut, std::string& hostOut); -int64_t atoi64(const char* psz); int64_t atoi64(const std::string& str); int atoi(const std::string& str); @@ -100,6 +99,13 @@ NODISCARD bool ParseInt32(const std::string& str, int32_t *out); NODISCARD bool ParseInt64(const std::string& str, int64_t *out); /** + * Convert decimal string to unsigned 8-bit integer with strict parse error feedback. + * @returns true if the entire string could be parsed as valid integer, + * false if not the entire string could be parsed or when overflow or underflow occurred. + */ +NODISCARD bool ParseUInt8(const std::string& str, uint8_t *out); + +/** * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. diff --git a/src/util/system.cpp b/src/util/system.cpp index 7b74789b32..00aa53df70 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -48,12 +48,6 @@ #pragma warning(disable:4717) #endif -#ifdef _WIN32_IE -#undef _WIN32_IE -#endif -#define _WIN32_IE 0x0501 - -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 4c1fe57c66..bf05ef844a 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -49,6 +49,7 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data. 0 to entirely disable the fallbackfee feature. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u). Warning: Smaller sizes may increase the risk of losing funds when restoring from an old backup, if none of the addresses in the original keypool have been used.", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + argsman.AddArg("-maxapsfee=<n>", strprintf("Spend up to this amount in additional (absolute) fees (in %s) if it allows the use of partial spend avoidance (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MAX_AVOIDPARTIALSPEND_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 7a03c4f544..17512265b5 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2836,6 +2836,15 @@ static UniValue listunspent(const JSONRPCRequest& request) if (!request.params[4].isNull()) { const UniValue& options = request.params[4].get_obj(); + RPCTypeCheckObj(options, + { + {"minimumAmount", UniValueType()}, + {"maximumAmount", UniValueType()}, + {"minimumSumAmount", UniValueType()}, + {"maximumCount", UniValueType(UniValue::VNUM)}, + }, + true, true); + if (options.exists("minimumAmount")) nMinimumAmount = AmountFromValue(options["minimumAmount"]); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 9c7b446c23..fa00d12551 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -276,7 +276,7 @@ std::string COutput::ToString() const const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { - LOCK(cs_wallet); + AssertLockHeld(cs_wallet); std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash); if (it == mapWallet.end()) return nullptr; @@ -1210,15 +1210,13 @@ void CWallet::BlockUntilSyncedToCurrentChain() const { isminetype CWallet::IsMine(const CTxIn &txin) const { + AssertLockHeld(cs_wallet); + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); + if (mi != mapWallet.end()) { - LOCK(cs_wallet); - std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); - if (mi != mapWallet.end()) - { - const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.tx->vout.size()) - return IsMine(prev.tx->vout[txin.prevout.n]); - } + const CWalletTx& prev = (*mi).second; + if (txin.prevout.n < prev.tx->vout.size()) + return IsMine(prev.tx->vout[txin.prevout.n]); } return ISMINE_NO; } @@ -1243,16 +1241,19 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const isminetype CWallet::IsMine(const CTxOut& txout) const { + AssertLockHeld(cs_wallet); return IsMine(txout.scriptPubKey); } isminetype CWallet::IsMine(const CTxDestination& dest) const { + AssertLockHeld(cs_wallet); return IsMine(GetScriptForDestination(dest)); } isminetype CWallet::IsMine(const CScript& script) const { + AssertLockHeld(cs_wallet); isminetype result = ISMINE_NO; for (const auto& spk_man_pair : m_spk_managers) { result = std::max(result, spk_man_pair.second->IsMine(script)); @@ -1264,6 +1265,7 @@ CAmount CWallet::GetCredit(const CTxOut& txout, const isminefilter& filter) cons { if (!MoneyRange(txout.nValue)) throw std::runtime_error(std::string(__func__) + ": value out of range"); + LOCK(cs_wallet); return ((IsMine(txout) & filter) ? txout.nValue : 0); } @@ -1281,13 +1283,12 @@ bool CWallet::IsChange(const CScript& script) const // a better way of identifying which outputs are 'the send' and which are // 'the change' will need to be implemented (maybe extend CWalletTx to remember // which output, if any, was change). + AssertLockHeld(cs_wallet); if (IsMine(script)) { CTxDestination address; if (!ExtractDestination(script, address)) return true; - - LOCK(cs_wallet); if (!FindAddressBookEntry(address)) { return true; } @@ -1297,6 +1298,7 @@ bool CWallet::IsChange(const CScript& script) const CAmount CWallet::GetChange(const CTxOut& txout) const { + AssertLockHeld(cs_wallet); if (!MoneyRange(txout.nValue)) throw std::runtime_error(std::string(__func__) + ": value out of range"); return (IsChange(txout) ? txout.nValue : 0); @@ -1304,6 +1306,7 @@ CAmount CWallet::GetChange(const CTxOut& txout) const bool CWallet::IsMine(const CTransaction& tx) const { + AssertLockHeld(cs_wallet); for (const CTxOut& txout : tx.vout) if (IsMine(txout)) return true; @@ -1362,6 +1365,7 @@ CAmount CWallet::GetCredit(const CTransaction& tx, const isminefilter& filter) c CAmount CWallet::GetChange(const CTransaction& tx) const { + LOCK(cs_wallet); CAmount nChange = 0; for (const CTxOut& txout : tx.vout) { @@ -1597,6 +1601,7 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, nFee = nDebit - nValueOut; } + LOCK(pwallet->cs_wallet); // Sent/received. for (unsigned int i = 0; i < tx->vout.size(); ++i) { @@ -1983,6 +1988,7 @@ bool CWalletTx::IsTrusted(std::set<uint256>& trusted_parents) const if (!InMempool()) return false; // Trusted if all inputs are from us and are in the mempool: + LOCK(pwallet->cs_wallet); for (const CTxIn& txin : tx->vin) { // Transactions not sent by us: not trusted @@ -2677,7 +2683,14 @@ OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_typ return m_default_address_type; } -bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign) +bool CWallet::CreateTransactionInternal( + const std::vector<CRecipient>& vecSend, + CTransactionRef& tx, + CAmount& nFeeRet, + int& nChangePosInOut, + bilingual_str& error, + const CCoinControl& coin_control, + bool sign) { CAmount nValue = 0; const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); @@ -3032,6 +3045,39 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransac return true; } +bool CWallet::CreateTransaction( + const std::vector<CRecipient>& vecSend, + CTransactionRef& tx, + CAmount& nFeeRet, + int& nChangePosInOut, + bilingual_str& error, + const CCoinControl& coin_control, + bool sign) +{ + int nChangePosIn = nChangePosInOut; + CTransactionRef tx2 = tx; + bool res = CreateTransactionInternal(vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, sign); + // try with avoidpartialspends unless it's enabled already + if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { + CCoinControl tmp_cc = coin_control; + tmp_cc.m_avoid_partial_spends = true; + CAmount nFeeRet2; + int nChangePosInOut2 = nChangePosIn; + bilingual_str error2; // fired and forgotten; if an error occurs, we discard the results + if (CreateTransactionInternal(vecSend, tx2, nFeeRet2, nChangePosInOut2, error2, tmp_cc, sign)) { + // if fee of this alternative one is within the range of the max fee, we use this one + const bool use_aps = nFeeRet2 <= nFeeRet + m_max_aps_fee; + WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", nFeeRet, nFeeRet2, use_aps ? "grouped" : "non-grouped"); + if (use_aps) { + tx = tx2; + nFeeRet = nFeeRet2; + nChangePosInOut = nChangePosInOut2; + } + } + } + return res; +} + void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { LOCK(cs_wallet); @@ -3154,6 +3200,7 @@ DBErrors CWallet::ZapWalletTx(std::list<CWalletTx>& vWtx) bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose) { bool fUpdated = false; + bool is_mine; { LOCK(cs_wallet); std::map<CTxDestination, CAddressBookData>::iterator mi = m_address_book.find(address); @@ -3161,8 +3208,9 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add m_address_book[address].SetLabel(strName); if (!strPurpose.empty()) /* update purpose only if requested */ m_address_book[address].purpose = strPurpose; + is_mine = IsMine(address) != ISMINE_NO; } - NotifyAddressBookChanged(this, address, strName, IsMine(address) != ISMINE_NO, + NotifyAddressBookChanged(this, address, strName, is_mine, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose)) return false; @@ -3177,17 +3225,16 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s bool CWallet::DelAddressBook(const CTxDestination& address) { - // If we want to delete receiving addresses, we need to take care that DestData "used" (and possibly newer DestData) gets preserved (and the "deleted" address transformed into a change entry instead of actually being deleted) - // NOTE: This isn't a problem for sending addresses because they never have any DestData yet! - // When adding new DestData, it should be considered here whether to retain or delete it (or move it?). - if (IsMine(address)) { - WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, PACKAGE_BUGREPORT); - return false; - } - + bool is_mine; { LOCK(cs_wallet); - + // If we want to delete receiving addresses, we need to take care that DestData "used" (and possibly newer DestData) gets preserved (and the "deleted" address transformed into a change entry instead of actually being deleted) + // NOTE: This isn't a problem for sending addresses because they never have any DestData yet! + // When adding new DestData, it should be considered here whether to retain or delete it (or move it?). + if (IsMine(address)) { + WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, PACKAGE_BUGREPORT); + return false; + } // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); for (const std::pair<const std::string, std::string> &item : m_address_book[address].destdata) @@ -3195,9 +3242,10 @@ bool CWallet::DelAddressBook(const CTxDestination& address) WalletBatch(*database).EraseDestData(strAddress, item.first); } m_address_book.erase(address); + is_mine = IsMine(address) != ISMINE_NO; } - NotifyAddressBookChanged(this, address, "", IsMine(address) != ISMINE_NO, "", CT_DELETED); + NotifyAddressBookChanged(this, address, "", is_mine, "", CT_DELETED); WalletBatch(*database).ErasePurpose(EncodeDestination(address)); return WalletBatch(*database).EraseName(EncodeDestination(address)); @@ -3849,6 +3897,22 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->m_min_fee = CFeeRate(n); } + if (gArgs.IsArgSet("-maxapsfee")) { + const std::string max_aps_fee{gArgs.GetArg("-maxapsfee", "")}; + CAmount n = 0; + if (max_aps_fee == "-1") { + n = -1; + } else if (!ParseMoney(max_aps_fee, n)) { + error = AmountErrMsg("maxapsfee", max_aps_fee); + return nullptr; + } + if (n > HIGH_APS_FEE) { + warnings.push_back(AmountHighWarn("-maxapsfee") + Untranslated(" ") + + _("This is the maximum transaction fee you pay (in addition to the normal fee) to prioritize partial spend avoidance over regular coin selection.")); + } + walletInstance->m_max_aps_fee = n; + } + if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index a761caf38c..f421de0cf2 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -72,6 +72,16 @@ static const CAmount DEFAULT_FALLBACK_FEE = 0; static const CAmount DEFAULT_DISCARD_FEE = 10000; //! -mintxfee default static const CAmount DEFAULT_TRANSACTION_MINFEE = 1000; +/** + * maximum fee increase allowed to do partial spend avoidance, even for nodes with this feature disabled by default + * + * A value of -1 disables this feature completely. + * A value of 0 (current default) means to attempt to do partial spend avoidance, and use its results if the fees remain *unchanged* + * A value > 0 means to do partial spend avoidance if the fee difference against a regular coin selection instance is in the range [0..value]. + */ +static const CAmount DEFAULT_MAX_AVOIDPARTIALSPEND_FEE = 0; +//! discourage APS fee higher than this amount +constexpr CAmount HIGH_APS_FEE{COIN / 10000}; //! minimum recommended increment for BIP 125 replacement txs static const CAmount WALLET_INCREMENTAL_RELAY_FEE = 5000; //! Default for -spendzeroconfchange @@ -217,7 +227,7 @@ static inline void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) nOrderPos = -1; // TODO: calculate elsewhere return; } - nOrderPos = atoi64(mapValue["n"].c_str()); + nOrderPos = atoi64(mapValue["n"]); } @@ -719,6 +729,8 @@ private: // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + bool CreateTransactionInternal(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign); + public: /* * Main wallet lock. @@ -793,7 +805,7 @@ public: /** Interface for accessing chain state. */ interfaces::Chain& chain() const { assert(m_chain); return *m_chain; } - const CWalletTx* GetWalletTx(const uint256& hash) const; + const CWalletTx* GetWalletTx(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! check whether we are allowed to upgrade (or already support) to the named feature bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } @@ -1008,6 +1020,7 @@ public: */ CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; + CAmount m_max_aps_fee{DEFAULT_MAX_AVOIDPARTIALSPEND_FEE}; //!< note: this is absolute fee, not fee rate OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; /** * Default output type for change outputs. When unset, automatically choose type @@ -1038,20 +1051,20 @@ public: bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error); - isminetype IsMine(const CTxDestination& dest) const; - isminetype IsMine(const CScript& script) const; - isminetype IsMine(const CTxIn& txin) const; + isminetype IsMine(const CTxDestination& dest) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + isminetype IsMine(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + isminetype IsMine(const CTxIn& txin) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Returns amount of debit if the input matches the * filter, otherwise returns 0 */ CAmount GetDebit(const CTxIn& txin, const isminefilter& filter) const; - isminetype IsMine(const CTxOut& txout) const; + isminetype IsMine(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); CAmount GetCredit(const CTxOut& txout, const isminefilter& filter) const; - bool IsChange(const CTxOut& txout) const; - bool IsChange(const CScript& script) const; - CAmount GetChange(const CTxOut& txout) const; - bool IsMine(const CTransaction& tx) const; + bool IsChange(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsChange(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + CAmount GetChange(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsMine(const CTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** should probably be renamed to IsRelevantToMe */ bool IsFromMe(const CTransaction& tx) const; CAmount GetDebit(const CTransaction& tx, const isminefilter& filter) const; diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index c1cba0fd13..9b51461843 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -130,7 +130,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) std::vector<bilingual_str> warnings; bool ret = RecoverDatabaseFile(path, error, warnings); if (!ret) { - for (const auto warning : warnings) { + for (const auto& warning : warnings) { tfm::format(std::cerr, "%s\n", warning.original); } if (!error.empty()) { diff --git a/test/functional/README.md b/test/functional/README.md index aff5f714f2..0d85a74074 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -127,8 +127,8 @@ Base class for functional tests. #### [util.py](test_framework/util.py) Generally useful functions. -#### [mininode.py](test_framework/mininode.py) -Basic code to support P2P connectivity to a bitcoind. +#### [p2p.py](test_framework/p2p.py) +Test objects for interacting with a bitcoind node over the p2p interface. #### [script.py](test_framework/script.py) Utilities for manipulating transaction scripts (originally from python-bitcoinlib) diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 34e4999329..1832043989 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -16,17 +16,16 @@ from collections import defaultdict # Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv, MSG_BLOCK -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, msg_block, msg_getdata, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - wait_until, ) # P2PInterface is a class containing callbacks to be executed when a P2P @@ -167,7 +166,7 @@ class ExampleTest(BitcoinTestFramework): height = self.nodes[0].getblockcount() for _ in range(10): - # Use the mininode and blocktools functionality to manually build a block + # Use the blocktools functionality to manually build a block. # Calling the generate() rpc is easier, but this allows us to exactly # control the blocks and transactions. block = create_block(self.tip, create_coinbase(height+1), self.block_time) @@ -203,13 +202,13 @@ class ExampleTest(BitcoinTestFramework): # wait_until() will loop until a predicate condition is met. Use it to test properties of the # P2PInterface objects. - wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock) + self.nodes[2].p2p.wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5) self.log.info("Check that each block was received only once") # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate. - with mininode_lock: + with p2p_lock: for block in self.nodes[2].p2p.block_receive_map.values(): assert_equal(block, 1) diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 75267de80b..17fbf50cc8 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -11,7 +11,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until, get_datadir_path, connect_nodes +from test_framework.util import get_datadir_path, connect_nodes import os @@ -41,7 +41,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=200) + self.nodes[0].wait_until_stopped(timeout=200) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index f19ee12f95..603d7f5d3b 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -42,7 +42,7 @@ from test_framework.messages import ( msg_block, msg_headers, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index c74761869b..efafcfaec3 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -26,7 +26,7 @@ from test_framework.messages import ( uint256_from_compact, uint256_from_str, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, MAX_SCRIPT_ELEMENT_SIZE, @@ -53,7 +53,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from data import invalid_txs -# Use this class for tests that require behavior other than normal "mininode" behavior. +# Use this class for tests that require behavior other than normal p2p behavior. # For now, it is used to serialize a bloated varint (b64). class CBrokenBlock(CBlock): def initialize(self, base_block): diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index fd0330924d..2919b0ea0b 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -10,7 +10,7 @@ Test that the CHECKLOCKTIMEVERIFY soft-fork activates at (regtest) block height from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import CTransaction, msg_block, ToHex -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP, CScriptNum from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index dfb3683143..38e95f00e9 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -44,7 +44,7 @@ import time from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import ToHex, CTransaction -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, OP_CHECKSEQUENCEVERIFY, diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 05fdacd451..f263c93c8a 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -9,7 +9,7 @@ Test that the DERSIG soft-fork activates at (regtest) height 1251. from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import msg_block -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 0dc2839191..e5c62d1ea7 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -14,7 +14,7 @@ from collections import defaultdict import time from test_framework.messages import CInv, MSG_BLOCK, msg_getdata -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, mine_large_block diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index dd4c318cee..3497b49a19 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, connect_nodes, disconnect_nodes, hex_str_to_bytes, @@ -56,7 +55,7 @@ class NotificationsTest(BitcoinTestFramework): blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) # wait at most 10 seconds for expected number of files before reading the content - wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) # directory content should equal the generated blocks hashes assert_equal(sorted(blocks), sorted(os.listdir(self.blocknotify_dir))) @@ -64,7 +63,7 @@ class NotificationsTest(BitcoinTestFramework): if self.is_wallet_compiled(): self.log.info("test -walletnotify") # wait at most 10 seconds for expected number of files before reading the content - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) @@ -78,7 +77,7 @@ class NotificationsTest(BitcoinTestFramework): self.start_node(1) connect_nodes(self.nodes[0], 1) - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) @@ -140,7 +139,7 @@ class NotificationsTest(BitcoinTestFramework): # TODO: add test for `-alertnotify` large fork notifications def expect_wallet_notify(self, tx_ids): - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) assert_equal(sorted(notify_outputname(self.wallet, tx_id) for tx_id in tx_ids), sorted(os.listdir(self.walletnotify_dir))) for tx_file in os.listdir(self.walletnotify_dir): os.remove(os.path.join(self.walletnotify_dir, tx_file)) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 02fa88f7c8..db408ab67a 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -20,7 +20,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) # Rescans start at the earliest block up to 2 hours before a key timestamp, so @@ -136,7 +135,7 @@ class PruneTest(BitcoinTestFramework): mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned - wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) + self.wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) self.log.info("Success") usage = calc_usage(self.prunedir) @@ -250,7 +249,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height - wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) + self.wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py index d782d3b1d8..a76e0f1b50 100755 --- a/test/functional/feature_shutdown.py +++ b/test/functional/feature_shutdown.py @@ -5,7 +5,7 @@ """Test bitcoind shutdown.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_rpc_proxy, wait_until +from test_framework.util import assert_equal, get_rpc_proxy from threading import Thread def test_long_call(node): @@ -25,7 +25,7 @@ class ShutdownTest(BitcoinTestFramework): node.getblockcount() Thread(target=test_long_call, args=(node,)).start() # Wait until the server is executing the above `waitfornewblock`. - wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) + self.wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) # Wait 1 second after requesting shutdown but not before the `stop` call # finishes. This is to ensure event loop waits for current connections # to close. diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 0713925141..e045adac32 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -12,9 +12,8 @@ import re from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import msg_block -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until VB_PERIOD = 144 # versionbits period length for regtest VB_THRESHOLD = 108 # versionbits activation threshold for regtest @@ -91,14 +90,14 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one block guarantees that we'll get out of IBD node.generatetoaddress(1, node_deterministic_address) - wait_until(lambda: not node.getblockchaininfo()['initialblockdownload'], timeout=10, lock=mininode_lock) + self.wait_until(lambda: not node.getblockchaininfo()['initialblockdownload'], timeout=10, lock=p2p_lock) # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning - wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) + self.wait_until(lambda: self.versionbits_in_alert_file()) if __name__ == '__main__': VersionBitsWarningTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 98dac30ace..e74ef8cf16 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,13 +7,12 @@ from decimal import Decimal from test_framework.messages import COIN -from test_framework.mininode import P2PTxInvStore +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, satoshi_round, - wait_until, ) # default limits @@ -269,8 +268,8 @@ class MempoolPackagesTest(BitcoinTestFramework): # - txs from previous ancestor test (-> custom ancestor limit) # - parent tx for descendant test # - txs chained off parent tx (-> custom descendant limit) - wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == - MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == + MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) mempool0 = self.nodes[0].getrawmempool(False) mempool1 = self.nodes[1].getrawmempool(False) assert set(mempool1).issubset(set(mempool0)) diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 85c4d6d570..f73f1a02a2 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -39,15 +39,14 @@ from decimal import Decimal import os import time +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.mininode import P2PTxInvStore from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) @@ -172,7 +171,7 @@ class MempoolPersistTest(BitcoinTestFramework): # check that txn gets broadcast due to unbroadcast logic conn = node0.add_p2p_connection(P2PTxInvStore()) node0.mockscheduler(16*60) # 15 min + 1 for buffer - wait_until(lambda: len(conn.get_invs()) == 1) + self.wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 365d011157..abd5a03d95 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -7,7 +7,7 @@ to peers until a GETDATA is received.""" import time -from test_framework.mininode import P2PTxInvStore +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 63d1ccfb36..b13740750f 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -20,7 +20,7 @@ from test_framework.messages import ( CBlockHeader, BLOCK_HEADER_SIZE, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 5c7e27a3a8..80f262d0d3 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -12,9 +12,7 @@ from test_framework.messages import ( NODE_WITNESS, msg_addr, ) -from test_framework.mininode import ( - P2PInterface, -) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index a9e86bd2fc..84178d0dd7 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -18,13 +18,12 @@ from test_framework.messages import ( ser_uint256, uint256_from_str, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, disconnect_nodes, - wait_until, ) class CFiltersClient(P2PInterface): @@ -65,11 +64,11 @@ class CompactFiltersTest(BitcoinTestFramework): disconnect_nodes(self.nodes[0], 1) self.nodes[0].generate(1) - wait_until(lambda: self.nodes[0].getblockcount() == 1000) + self.wait_until(lambda: self.nodes[0].getblockcount() == 1000) stale_block_hash = self.nodes[0].getblockhash(1000) self.nodes[1].generate(1001) - wait_until(lambda: self.nodes[1].getblockcount() == 2000) + self.wait_until(lambda: self.nodes[1].getblockcount() == 2000) # Check that nodes have signalled NODE_COMPACT_FILTERS correctly. assert node0.nServices & NODE_COMPACT_FILTERS != 0 diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 27e6b669f6..65259f1869 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -5,7 +5,7 @@ """Test p2p blocksonly""" from test_framework.messages import msg_tx, CTransaction, FromHex -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 225d393e1b..fdae7fb68b 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -11,10 +11,10 @@ import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, wait_until, softfork_active +from test_framework.util import assert_equal, softfork_active # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): @@ -48,12 +48,12 @@ class TestP2PConn(P2PInterface): self.block_announced = True self.announced_blockhashes.add(x.hash) - # Requires caller to hold mininode_lock + # Requires caller to hold p2p_lock def received_block_announcement(self): return self.block_announced def clear_block_announcement(self): - with mininode_lock: + with p2p_lock: self.block_announced = False self.last_message.pop("inv", None) self.last_message.pop("headers", None) @@ -73,7 +73,7 @@ class TestP2PConn(P2PInterface): def request_headers_and_sync(self, locator, hashstop=0): self.clear_block_announcement() self.get_headers(locator, hashstop) - wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock) + self.wait_until(self.received_block_announcement, timeout=30) self.clear_block_announcement() # Block until a block announcement for a particular block hash is @@ -81,7 +81,7 @@ class TestP2PConn(P2PInterface): def wait_for_block_announcement(self, block_hash, timeout=30): def received_hash(): return (block_hash in self.announced_blockhashes) - wait_until(received_hash, timeout=timeout, lock=mininode_lock) + self.wait_until(received_hash, timeout=timeout) def send_await_disconnect(self, message, timeout=30): """Sends a message to the node and wait for disconnect. @@ -89,7 +89,7 @@ class TestP2PConn(P2PInterface): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - wait_until(lambda: not self.is_connected, timeout=timeout, lock=mininode_lock) + self.wait_for_disconnect(timeout) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): @@ -154,8 +154,8 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) - wait_until(received_sendcmpct, timeout=30, lock=mininode_lock) - with mininode_lock: + test_node.wait_until(received_sendcmpct, timeout=30) + with p2p_lock: # Check that the first version received is the preferred one assert_equal(test_node.last_sendcmpct[0].version, preferred_version) # And that we receive versions down to 1. @@ -170,7 +170,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.wait_for_block_announcement(block_hash, timeout=30) assert peer.block_announced - with mininode_lock: + with p2p_lock: assert predicate(peer), ( "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) @@ -294,11 +294,11 @@ class CompactBlocksTest(BitcoinTestFramework): block.rehash() # Wait until the block was announced (via compact blocks) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) # Now fetch and check the compact block header_and_shortids = None - with mininode_lock: + with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -308,11 +308,11 @@ class CompactBlocksTest(BitcoinTestFramework): inv = CInv(MSG_CMPCT_BLOCK, block_hash) test_node.send_message(msg_getdata([inv])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) # Now fetch and check the compact block header_and_shortids = None - with mininode_lock: + with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -378,7 +378,7 @@ class CompactBlocksTest(BitcoinTestFramework): if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) - wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) @@ -397,7 +397,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. - with mininode_lock: + with p2p_lock: assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -439,7 +439,7 @@ class CompactBlocksTest(BitcoinTestFramework): def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -504,13 +504,13 @@ class CompactBlocksTest(BitcoinTestFramework): assert tx.hash in mempool # Clear out last request. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getblocktxn", None) # Send compact block comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) - with mininode_lock: + with p2p_lock: # Shouldn't have gotten a request for any transaction assert "getblocktxn" not in test_node.last_message @@ -537,7 +537,7 @@ class CompactBlocksTest(BitcoinTestFramework): comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] - with mininode_lock: + with p2p_lock: assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -588,10 +588,10 @@ class CompactBlocksTest(BitcoinTestFramework): num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) test_node.send_message(msg) - wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock) + test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) [tx.calc_sha256() for tx in block.vtx] - with mininode_lock: + with p2p_lock: assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: @@ -611,11 +611,11 @@ class CompactBlocksTest(BitcoinTestFramework): # allowed depth for a blocktxn response. block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) - with mininode_lock: + with p2p_lock: test_node.last_message.pop("block", None) test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) - with mininode_lock: + with p2p_lock: test_node.last_message["block"].block.calc_sha256() assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message @@ -628,21 +628,21 @@ class CompactBlocksTest(BitcoinTestFramework): for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): test_node.clear_block_announcement() new_blocks.append(node.generate(1)[0]) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) test_node.clear_block_announcement() node.generate(1) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() - with mininode_lock: + with p2p_lock: test_node.last_message.pop("block", None) test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) - wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) - with mininode_lock: + test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30) + with p2p_lock: test_node.last_message["block"].block.calc_sha256() assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) @@ -670,10 +670,10 @@ class CompactBlocksTest(BitcoinTestFramework): # (to avoid fingerprinting attacks). msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) - with mininode_lock: + with p2p_lock: test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "blocktxn" not in test_node.last_message def test_end_to_end_block_relay(self, listeners): @@ -689,8 +689,8 @@ class CompactBlocksTest(BitcoinTestFramework): node.submitblock(ToHex(block)) for l in listeners: - wait_until(lambda: "cmpctblock" in l.last_message, timeout=30, lock=mininode_lock) - with mininode_lock: + l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30) + with p2p_lock: for l in listeners: l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) @@ -747,7 +747,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.initialize_from_block(block) msg = msg_cmpctblock(cmpct_block.to_p2p()) peer.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "getblocktxn" in peer.last_message return block, cmpct_block diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 09b9ebeb2d..b7c2a306eb 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -10,7 +10,6 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, connect_nodes, - wait_until, ) class DisconnectBanTest(BitcoinTestFramework): @@ -28,7 +27,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: successfully ban single IP address") assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point self.nodes[1].setban(subnet="127.0.0.1", command="add") - wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point assert_equal(len(self.nodes[1].listbanned()), 1) @@ -95,7 +94,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] self.nodes[0].disconnectnode(address=address1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] self.log.info("disconnectnode: successfully reconnect node") @@ -106,7 +105,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by node id") id1 = self.nodes[0].getpeerinfo()[0]['id'] self.nodes[0].disconnectnode(nodeid=id1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] if __name__ == '__main__': diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index f8552cf53d..7dd8c3146b 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -8,7 +8,7 @@ from test_framework.messages import ( CBlockHeader, FromHex, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, msg_headers, ) diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index b2b3a89aab..72a255991c 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -15,11 +15,11 @@ Therefore, this test is limited to the remaining protection criteria. import time -from test_framework.test_framework import BitcoinTestFramework -from test_framework.mininode import P2PInterface, P2PDataStore -from test_framework.util import assert_equal, wait_until from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CTransaction, FromHex, msg_pong, msg_tx +from test_framework.p2p import P2PDataStore, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal class SlowP2PDataStore(P2PDataStore): @@ -92,7 +92,7 @@ class P2PEvict(BitcoinTestFramework): for _ in range(8): fastpeer = node.add_p2p_connection(P2PInterface()) current_peer += 1 - wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) + self.wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) # Make sure by asking the node what the actual min pings are peerinfo = node.getpeerinfo() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 3a9b8dfbd7..0c07b56a69 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -7,7 +7,7 @@ from decimal import Decimal from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import P2PInterface, p2p_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -23,7 +23,7 @@ class FeefilterConn(P2PInterface): self.feefilter_received = True def assert_feefilter_received(self, recv: bool): - with mininode_lock: + with p2p_lock: assert_equal(self.feefilter_received, recv) @@ -39,10 +39,10 @@ class TestP2PConn(P2PInterface): def wait_for_invs_to_match(self, invs_expected): invs_expected.sort() - self.wait_until(lambda: invs_expected == sorted(self.txinvs), timeout=60) + self.wait_until(lambda: invs_expected == sorted(self.txinvs)) def clear_invs(self): - with mininode_lock: + with p2p_lock: self.txinvs = [] diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index ce3856fc95..613d96eaad 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -19,7 +19,7 @@ from test_framework.messages import ( msg_mempool, msg_version, ) -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface, p2p_lock from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -60,22 +60,22 @@ class P2PBloomFilter(P2PInterface): @property def tx_received(self): - with mininode_lock: + with p2p_lock: return self._tx_received @tx_received.setter def tx_received(self, value): - with mininode_lock: + with p2p_lock: self._tx_received = value @property def merkleblock_received(self): - with mininode_lock: + with p2p_lock: return self._merkleblock_received @merkleblock_received.setter def merkleblock_received(self, value): - with mininode_lock: + with p2p_lock: self._merkleblock_received = value diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index d743abe681..aaf862e6c8 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -12,7 +12,7 @@ import time from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv, MSG_BLOCK -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, msg_headers, msg_block, @@ -22,9 +22,9 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) + class P2PFingerprintTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -102,12 +102,12 @@ class P2PFingerprintTest(BitcoinTestFramework): # Check that getdata request for stale block succeeds self.send_block_request(stale_hash, node0) test_function = lambda: self.last_block_equals(stale_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) # Check that getheader request for stale block header succeeds self.send_header_request(stale_hash, node0) test_function = lambda: self.last_header_equals(stale_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) # Longest chain is extended so stale is much older than chain tip self.nodes[0].setmocktime(0) @@ -138,11 +138,11 @@ class P2PFingerprintTest(BitcoinTestFramework): self.send_block_request(block_hash, node0) test_function = lambda: self.last_block_equals(block_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) self.send_header_request(block_hash, node0) test_function = lambda: self.last_header_equals(block_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) if __name__ == '__main__': P2PFingerprintTest().main() diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index c9278eab92..6622ea9ec2 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -12,9 +12,9 @@ from test_framework.messages import ( msg_addr, msg_getaddr, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock + p2p_lock ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -44,7 +44,7 @@ class AddrReceiver(P2PInterface): self.received_addrs = None def get_received_addrs(self): - with mininode_lock: + with p2p_lock: return self.received_addrs def on_addr(self, message): diff --git a/test/functional/p2p_getdata.py b/test/functional/p2p_getdata.py index d1b11c2c61..51921a8ab5 100755 --- a/test/functional/p2p_getdata.py +++ b/test/functional/p2p_getdata.py @@ -9,7 +9,7 @@ from test_framework.messages import ( CInv, msg_getdata, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index e280a62997..b2c3c5d45f 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -14,7 +14,7 @@ import copy from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script from test_framework.messages import COIN -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index 0155eb21f0..24328c2919 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -6,7 +6,7 @@ """ from test_framework.messages import msg_getheaders, msg_getblocks, MAX_LOCATOR_SZ -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index d9a9ae5188..fe57057a83 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -17,14 +17,13 @@ from test_framework.messages import ( MSG_TX, ser_string, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PDataStore, P2PInterface, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte length prefix @@ -70,7 +69,7 @@ class InvalidMessagesTest(BitcoinTestFramework): before = int(self.nodes[0].getnettotals()['totalbytesrecv']) conn.send_raw_message(msg[:cut_pos]) # Wait until node has processed the first half of the message - wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) + self.wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) middle = int(self.nodes[0].getnettotals()['totalbytesrecv']) # If this assert fails, we've hit an unlikely race # where the test framework sent a message in between the two halves diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index c70a892463..a0ef6c9d6e 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -13,11 +13,10 @@ from test_framework.messages import ( CTxIn, CTxOut, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) from data import invalid_txs @@ -146,7 +145,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): # tx_orphan_no_fee, because it has too low fee (p2ps[0] is not disconnected for relaying that tx) # tx_orphan_invaid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) - wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected + self.wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) self.log.info('Test orphan pool overflow') diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 2fc5245241..4978aa3845 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -15,21 +15,19 @@ import time from test_framework.messages import ( msg_getaddr, msg_ping, - msg_verack, msg_version, ) -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than_or_equal, - wait_until, ) DISCOURAGEMENT_THRESHOLD = 100 -class CLazyNode(P2PInterface): +class LazyPeer(P2PInterface): def __init__(self): super().__init__() self.unexpected_msg = False @@ -42,6 +40,7 @@ class CLazyNode(P2PInterface): def on_open(self): self.ever_connected = True + # Does not respond to "version" with "verack" def on_version(self, message): self.bad_message(message) def on_verack(self, message): self.bad_message(message) def on_inv(self, message): self.bad_message(message) @@ -64,21 +63,8 @@ class CLazyNode(P2PInterface): def on_blocktxn(self, message): self.bad_message(message) -# Node that never sends a version. We'll use this to send a bunch of messages -# anyway, and eventually get disconnected. -class CNodeNoVersionMisbehavior(CLazyNode): - pass - - -# Node that never sends a version. This one just sits idle and hopes to receive -# any message (it shouldn't!) -class CNodeNoVersionIdle(CLazyNode): - def __init__(self): - super().__init__() - - -# Node that sends a version but not a verack. -class CNodeNoVerackIdle(CLazyNode): +# Peer that sends a version but not a verack. +class NoVerackIdlePeer(LazyPeer): def __init__(self): self.version_received = False super().__init__() @@ -97,6 +83,7 @@ class P2PVersionStore(P2PInterface): version_received = None def on_version(self, msg): + # Responds with an appropriate verack super().on_version(msg) self.version_received = msg @@ -106,39 +93,45 @@ class P2PLeakTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): - no_version_disconnect_node = self.nodes[0].add_p2p_connection( - CNodeNoVersionMisbehavior(), send_version=False, wait_for_verack=False) - no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False) - no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False) + # Peer that never sends a version. We will send a bunch of messages + # from this peer anyway and verify eventual disconnection. + no_version_disconnect_peer = self.nodes[0].add_p2p_connection( + LazyPeer(), send_version=False, wait_for_verack=False) + + # Another peer that never sends a version, nor any other messages. It shouldn't receive anything from the node. + no_version_idle_peer = self.nodes[0].add_p2p_connection(LazyPeer(), send_version=False, wait_for_verack=False) + + # Peer that sends a version but not a verack. + no_verack_idle_peer = self.nodes[0].add_p2p_connection(NoVerackIdlePeer(), wait_for_verack=False) - # Send enough veracks without a message to reach the peer discouragement - # threshold. This should get us disconnected. + # Send enough ping messages (any non-version message will do) prior to sending + # version to reach the peer discouragement threshold. This should get us disconnected. for _ in range(DISCOURAGEMENT_THRESHOLD): - no_version_disconnect_node.send_message(msg_verack()) + no_version_disconnect_peer.send_message(msg_ping()) - # Wait until we got the verack in response to the version. Though, don't wait for the other node to receive the + # Wait until we got the verack in response to the version. Though, don't wait for the node to receive the # verack, since we never sent one - no_verack_idlenode.wait_for_verack() + no_verack_idle_peer.wait_for_verack() - wait_until(lambda: no_version_disconnect_node.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) + self.wait_until(lambda: no_version_disconnect_peer.ever_connected, timeout=10, lock=p2p_lock) + self.wait_until(lambda: no_version_idle_peer.ever_connected, timeout=10, lock=p2p_lock) + self.wait_until(lambda: no_verack_idle_peer.version_received, timeout=10, lock=p2p_lock) - # Mine a block and make sure that it's not sent to the connected nodes - self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address) + # Mine a block and make sure that it's not sent to the connected peers + self.nodes[0].generate(nblocks=1) #Give the node enough time to possibly leak out a message time.sleep(5) - # Expect this node to be disconnected for misbehavior - assert not no_version_disconnect_node.is_connected + # Expect this peer to be disconnected for misbehavior + assert not no_version_disconnect_peer.is_connected self.nodes[0].disconnect_p2ps() # Make sure no unexpected messages came in - assert no_version_disconnect_node.unexpected_msg == False - assert no_version_idlenode.unexpected_msg == False - assert no_verack_idlenode.unexpected_msg == False + assert no_version_disconnect_peer.unexpected_msg == False + assert no_version_idle_peer.unexpected_msg == False + assert no_verack_idle_peer.unexpected_msg == False self.log.info('Check that the version message does not leak the local address of the node') p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) @@ -151,13 +144,13 @@ class P2PLeakTest(BitcoinTestFramework): assert_equal(ver.nStartingHeight, 201) assert_equal(ver.nRelay, 1) - self.log.info('Check that old nodes are disconnected') - p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) + self.log.info('Check that old peers are disconnected') + p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) old_version_msg = msg_version() old_version_msg.nVersion = 31799 with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']): - p2p_old_node.send_message(old_version_msg) - p2p_old_node.wait_for_disconnect() + p2p_old_peer.send_message(old_version_msg) + p2p_old_peer.wait_for_disconnect() if __name__ == '__main__': diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index da30ad5977..9e761db03f 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -5,7 +5,7 @@ """Test that we don't leak txs to inbound peers that we haven't yet announced to""" from test_framework.messages import msg_getdata, CInv, MSG_TX -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_nobloomfilter_messages.py b/test/functional/p2p_nobloomfilter_messages.py index accc5dc23c..c2311cb197 100755 --- a/test/functional/p2p_nobloomfilter_messages.py +++ b/test/functional/p2p_nobloomfilter_messages.py @@ -12,7 +12,7 @@ Test that, when bloom filters are not enabled, peers are disconnected if: """ from test_framework.messages import msg_mempool, msg_filteradd, msg_filterload, msg_filterclear -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index a2f6ea538c..2c9cbea5e4 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -9,13 +9,12 @@ and that it responds to getdata requests for blocks correctly: - send a block within 288 + 2 of the tip - disconnect peers who request blocks older than that.""" from test_framework.messages import CInv, MSG_BLOCK, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, disconnect_nodes, connect_nodes, - wait_until, ) @@ -28,7 +27,7 @@ class P2PIgnoreInv(P2PInterface): self.firstAddrnServices = message.addrs[0].nServices def wait_for_addr(self, timeout=5): test_function = lambda: self.last_message.get("addr") - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def send_getdata_for_block(self, blockhash): getdata_request = msg_getdata() getdata_request.inv.append(CInv(MSG_BLOCK, int(blockhash, 16))) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 254352c816..3ec36edb41 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -13,7 +13,7 @@ from test_framework.messages import ( CTxInWitness, FromHex, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, OP_TRUE, @@ -24,7 +24,6 @@ from test_framework.util import ( assert_equal, connect_nodes, p2p_port, - wait_until, ) @@ -109,7 +108,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.sync_all() self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") - # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only + # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would # currently not request a txid that is already in the mempool. self.restart_node(1, extra_args=["-whitelist=forcerelay@127.0.0.1"]) @@ -137,7 +136,7 @@ class P2PPermissionsTests(BitcoinTestFramework): connect_nodes(self.nodes[1], 0) with self.nodes[1].assert_debug_log(["Force relaying tx {} from peer=0".format(txid)]): p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) - wait_until(lambda: txid in self.nodes[0].getrawmempool()) + self.wait_until(lambda: txid in self.nodes[0].getrawmempool()) self.log.debug("Check that node[1] will not send an invalid tx to node[0]") tx.vout[0].nValue += 1 diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py index 5f5fd3e104..888e986fba 100755 --- a/test/functional/p2p_ping.py +++ b/test/functional/p2p_ping.py @@ -8,7 +8,7 @@ import time from test_framework.messages import msg_pong -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 564e49f3d8..2155c1d0e7 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -42,9 +42,9 @@ from test_framework.messages import ( uint256_from_str, FromHex, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, + p2p_lock, ) from test_framework.script import ( CScript, @@ -83,7 +83,6 @@ from test_framework.util import ( softfork_active, hex_str_to_bytes, assert_raises_rpc_error, - wait_until, ) # The versionbit bit used to signal activation of SegWit @@ -153,8 +152,8 @@ class TestP2PConn(P2PInterface): self.lastgetdata = [] self.wtxidrelay = wtxidrelay - # Avoid sending out msg_getdata in the mininode thread as a reply to invs. - # They are not needed and would only lead to races because we send msg_getdata out in the test thread + # Don't send getdata message replies to invs automatically. + # We'll send the getdata messages explicitly in the test logic. def on_inv(self, message): pass @@ -177,7 +176,7 @@ class TestP2PConn(P2PInterface): if success: # sanity check assert (self.wtxidrelay and use_wtxid) or (not self.wtxidrelay and not use_wtxid) - with mininode_lock: + with p2p_lock: self.last_message.pop("getdata", None) if use_wtxid: wtxid = tx.calc_sha256(True) @@ -195,7 +194,7 @@ class TestP2PConn(P2PInterface): assert not self.last_message.get("getdata") def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): - with mininode_lock: + with p2p_lock: self.last_message.pop("getdata", None) self.last_message.pop("getheaders", None) msg = msg_headers() @@ -209,7 +208,7 @@ class TestP2PConn(P2PInterface): self.wait_for_getdata([block.sha256]) def request_block(self, blockhash, inv_type, timeout=60): - with mininode_lock: + with p2p_lock: self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) self.wait_for_block(blockhash, timeout) @@ -2114,7 +2113,7 @@ class SegWitTest(BitcoinTestFramework): # Check wtxidrelay feature negotiation message through connecting a new peer def received_wtxidrelay(): return (len(self.wtx_node.last_wtxidrelay) > 0) - wait_until(received_wtxidrelay, timeout=60, lock=mininode_lock) + self.wtx_node.wait_until(received_wtxidrelay) # Create a Segwit output from the latest UTXO # and announce it to the network @@ -2138,25 +2137,25 @@ class SegWitTest(BitcoinTestFramework): # Announce Segwit transaction with wtxid # and wait for getdata self.wtx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=True) - with mininode_lock: + with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_WTX, tx2.calc_sha256(True))]) # Announce Segwit transaction from non wtxidrelay peer # and wait for getdata self.tx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=False) - with mininode_lock: + with p2p_lock: lgd = self.tx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx2.sha256)]) # Send tx2 through; it's an orphan so won't be accepted - with mininode_lock: + with p2p_lock: self.wtx_node.last_message.pop("getdata", None) test_transaction_acceptance(self.nodes[0], self.wtx_node, tx2, with_witness=True, accepted=False) # Expect a request for parent (tx) by txid despite use of WTX peer self.wtx_node.wait_for_getdata([tx.sha256], 60) - with mininode_lock: + with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx.sha256)]) diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 126a46bd53..04e6ec4172 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -87,11 +87,11 @@ e. Announce one more that doesn't connect. """ from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CInv -from test_framework.mininode import ( +from test_framework.p2p import ( CBlockHeader, NODE_WITNESS, P2PInterface, - mininode_lock, + p2p_lock, MSG_BLOCK, msg_block, msg_getblocks, @@ -104,7 +104,6 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) DIRECT_FETCH_RESPONSE_TIME = 0.05 @@ -147,7 +146,7 @@ class BaseNode(P2PInterface): def wait_for_block_announcement(self, block_hash, timeout=60): test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def on_inv(self, message): self.block_announced = True @@ -163,7 +162,7 @@ class BaseNode(P2PInterface): self.last_blockhash_announced = message.headers[-1].sha256 def clear_block_announcements(self): - with mininode_lock: + with p2p_lock: self.block_announced = False self.last_message.pop("inv", None) self.last_message.pop("headers", None) @@ -174,8 +173,8 @@ class BaseNode(P2PInterface): """Test whether the last headers announcements received are right. Headers may be announced across more than one message.""" test_function = lambda: (len(self.recent_headers_announced) >= len(headers)) - wait_until(test_function, timeout=60, lock=mininode_lock) - with mininode_lock: + self.wait_until(test_function) + with p2p_lock: assert_equal(self.recent_headers_announced, headers) self.block_announced = False self.last_message.pop("headers", None) @@ -186,9 +185,9 @@ class BaseNode(P2PInterface): inv should be a list of block hashes.""" test_function = lambda: self.block_announced - wait_until(test_function, timeout=60, lock=mininode_lock) + self.wait_until(test_function) - with mininode_lock: + with p2p_lock: compare_inv = [] if "inv" in self.last_message: compare_inv = [x.hash for x in self.last_message["inv"].inv] @@ -298,7 +297,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks([new_block]) test_node.wait_for_getdata([new_block.sha256]) test_node.send_and_ping(msg_block(new_block)) # make sure this block is processed - wait_until(lambda: inv_node.block_announced, timeout=60, lock=mininode_lock) + inv_node.wait_until(lambda: inv_node.block_announced) inv_node.clear_block_announcements() test_node.clear_block_announcements() @@ -456,7 +455,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() # should not have received any getdata messages - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message # This time, direct fetch should work @@ -494,7 +493,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[0:1]) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message # Announcing one more block on fork should trigger direct fetch for @@ -513,7 +512,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[18:19]) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message self.log.info("Part 4: success!") @@ -536,7 +535,7 @@ class SendHeadersTest(BitcoinTestFramework): block_time += 1 height += 1 # Send the header of the second block -> this won't connect. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) test_node.wait_for_getheaders() @@ -559,7 +558,7 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_UNCONNECTING_HEADERS): # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) test_node.wait_for_getheaders() @@ -574,7 +573,7 @@ class SendHeadersTest(BitcoinTestFramework): # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i % len(blocks)]]) test_node.wait_for_getheaders() diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 5a4fa42988..ce12ce26ce 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -24,7 +24,7 @@ from time import sleep from test_framework.messages import msg_ping -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 3ea1c6e5e7..653c7ae43f 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -16,14 +16,13 @@ from test_framework.messages import ( msg_inv, msg_notfound, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE @@ -73,14 +72,14 @@ class TxDownloadTest(BitcoinTestFramework): def getdata_found(peer_index): p = self.nodes[0].p2ps[peer_index] - with mininode_lock: + with p2p_lock: return p.last_message.get("getdata") and p.last_message["getdata"].inv[-1].hash == txid node_0_mocktime = int(time.time()) while outstanding_peer_index: node_0_mocktime += MAX_GETDATA_INBOUND_WAIT self.nodes[0].setmocktime(node_0_mocktime) - wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) + self.wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) for i in outstanding_peer_index: if getdata_found(i): outstanding_peer_index.remove(i) @@ -134,24 +133,24 @@ class TxDownloadTest(BitcoinTestFramework): p = self.nodes[0].p2ps[0] - with mininode_lock: + with p2p_lock: p.tx_getdata_count = 0 p.send_message(msg_inv([CInv(t=MSG_WTX, h=i) for i in txids])) - wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) - with mininode_lock: + p.wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT) + with p2p_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info("Now check that if we send a NOTFOUND for a transaction, we'll get one more request") p.send_message(msg_notfound(vec=[CInv(t=MSG_WTX, h=txids[0])])) - wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) - with mininode_lock: + p.wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10) + with p2p_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) WAIT_TIME = TX_EXPIRY_INTERVAL // 2 + TX_EXPIRY_INTERVAL self.log.info("if we wait about {} minutes, we should eventually get more requests".format(WAIT_TIME / 60)) self.nodes[0].setmocktime(int(time.time() + WAIT_TIME)) - wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) + p.wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) self.nodes[0].setmocktime(0) def test_spurious_notfound(self): diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 71b0b0f63a..36b434bce3 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -55,7 +55,7 @@ import time from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script from test_framework.messages import CBlockHeader, CInv, MSG_BLOCK, msg_block, msg_headers, msg_inv -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -199,13 +199,13 @@ class AcceptBlockTest(BitcoinTestFramework): # 6. Try to get node to request the missing block. # Poke the node with an inv for block at height 3 and see if that # triggers a getdata on block 2 (it should if block 2 is missing). - with mininode_lock: + with p2p_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) test_node.send_message(msg_inv([CInv(MSG_BLOCK, block_h3.sha256)])) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 7c70f30ca3..c005584485 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -22,6 +22,17 @@ from decimal import Decimal import http.client import subprocess +from test_framework.blocktools import ( + create_block, + create_coinbase, + TIME_GENESIS_BLOCK, +) +from test_framework.messages import ( + CBlockHeader, + FromHex, + msg_block, +) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -32,17 +43,6 @@ from test_framework.util import ( assert_is_hex_string, assert_is_hash_string, ) -from test_framework.blocktools import ( - create_block, - create_coinbase, - TIME_GENESIS_BLOCK, -) -from test_framework.messages import ( - msg_block, -) -from test_framework.mininode import ( - P2PInterface, -) class BlockchainTest(BitcoinTestFramework): @@ -280,6 +280,14 @@ class BlockchainTest(BitcoinTestFramework): assert isinstance(int(header['versionHex'], 16), int) assert isinstance(header['difficulty'], Decimal) + # Test with verbose=False, which should return the header as hex. + header_hex = node.getblockheader(blockhash=besthash, verbose=False) + assert_is_hex_string(header_hex) + + header = FromHex(CBlockHeader(), header_hex) + header.calc_sha256() + assert_equal(header.hash, besthash) + def _test_getdifficulty(self): difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index 1fdc134f97..e788e75557 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR from test_framework.util import ( assert_equal, connect_nodes, - wait_until, ) @@ -57,9 +56,9 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("..and then mine a block") self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address) self.log.info("Verify all nodes are at the right height") - wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) - wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) - wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) + self.wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) + self.wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) + self.wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index cc5a264adb..0493ceeb64 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -61,6 +61,34 @@ class RpcMiscTest(BitcoinTestFramework): node.logging(include=['qt']) assert_equal(node.logging()['qt'], True) + self.log.info("test getindexinfo") + # Without any indices running the RPC returns an empty object + assert_equal(node.getindexinfo(), {}) + + # Restart the node with indices and wait for them to sync + self.restart_node(0, ["-txindex", "-blockfilterindex"]) + self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) + + # Returns a list of all running indices by default + assert_equal( + node.getindexinfo(), + { + "txindex": {"synced": True, "best_block_height": 200}, + "basic block filter index": {"synced": True, "best_block_height": 200} + } + ) + + # Specifying an index by name returns only the status of that index + assert_equal( + node.getindexinfo("txindex"), + { + "txindex": {"synced": True, "best_block_height": 200}, + } + ) + + # Specifying an unknown index name returns an empty result + assert_equal(node.getindexinfo("foo"), {}) + if __name__ == '__main__': RpcMiscTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 192b60e5d2..506c77c567 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -8,22 +8,24 @@ Tests correspond to code in rpc/net.cpp. """ from decimal import Decimal +from itertools import product +import time +from test_framework.p2p import P2PInterface +import test_framework.messages +from test_framework.messages import ( + NODE_NETWORK, + NODE_WITNESS, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( + assert_approx, assert_equal, assert_greater_than_or_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, p2p_port, - wait_until, -) -from test_framework.mininode import P2PInterface -import test_framework.messages -from test_framework.messages import ( - NODE_NETWORK, - NODE_WITNESS, ) @@ -48,25 +50,27 @@ class NetTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): - self.log.info('Get out of IBD for the minfeefilter test') - self.nodes[0].generate(1) - self.log.info('Connect nodes both way') + # Get out of IBD for the minfeefilter and getpeerinfo tests. + self.nodes[0].generate(101) + # Connect nodes both ways. connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) - self._test_connection_count() - self._test_getnettotals() - self._test_getnetworkinfo() - self._test_getaddednodeinfo() - self._test_getpeerinfo() + self.test_connection_count() + self.test_getpeerinfo() + self.test_getnettotals() + self.test_getnetworkinfo() + self.test_getaddednodeinfo() self.test_service_flags() - self._test_getnodeaddresses() + self.test_getnodeaddresses() - def _test_connection_count(self): - # connect_nodes connects each node to the other + def test_connection_count(self): + self.log.info("Test getconnectioncount") + # After using `connect_nodes` to connect nodes 0 and 1 to each other. assert_equal(self.nodes[0].getconnectioncount(), 2) - def _test_getnettotals(self): + def test_getnettotals(self): + self.log.info("Test getnettotals") # getnettotals totalbytesrecv and totalbytessent should be # consistent with getpeerinfo. Since the RPC calls are not atomic, # and messages might have been recvd or sent between RPC calls, call @@ -88,15 +92,16 @@ class NetTest(BitcoinTestFramework): # the bytes sent/received should change # note ping and pong are 32 bytes each self.nodes[0].ping() - wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) - wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): assert_greater_than_or_equal(after['bytesrecv_per_msg'].get('pong', 0), before['bytesrecv_per_msg'].get('pong', 0) + 32) assert_greater_than_or_equal(after['bytessent_per_msg'].get('ping', 0), before['bytessent_per_msg'].get('ping', 0) + 32) - def _test_getnetworkinfo(self): + def test_getnetworkinfo(self): + self.log.info("Test getnetworkinfo") assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) @@ -104,11 +109,11 @@ class NetTest(BitcoinTestFramework): self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close - wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) + self.wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.nodes[0].setnetworkactive(state=True) - self.log.info('Connect nodes both way') + # Connect nodes both ways. connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) @@ -120,7 +125,8 @@ class NetTest(BitcoinTestFramework): for info in network_info: assert_net_servicesnames(int(info["localservices"], 0x10), info["localservicesnames"]) - def _test_getaddednodeinfo(self): + def test_getaddednodeinfo(self): + self.log.info("Test getaddednodeinfo") assert_equal(self.nodes[0].getaddednodeinfo(), []) # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) @@ -139,8 +145,20 @@ class NetTest(BitcoinTestFramework): # check that a non-existent node returns an error assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') - def _test_getpeerinfo(self): + def test_getpeerinfo(self): + self.log.info("Test getpeerinfo") + # Create a few getpeerinfo last_block/last_transaction values. + if self.is_wallet_compiled(): + self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) + self.nodes[1].generate(1) + self.sync_all() + time_now = int(time.time()) peer_info = [x.getpeerinfo() for x in self.nodes] + # Verify last_block and last_transaction keys/values. + for node, peer, field in product(range(self.num_nodes), range(2), ['last_block', 'last_transaction']): + assert field in peer_info[node][peer].keys() + if peer_info[node][peer][field] != 0: + assert_approx(peer_info[node][peer][field], time_now, vspan=60) # check both sides of bidirectional connection between nodes # the address bound to on one side will be the source address for the other node assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) @@ -152,11 +170,13 @@ class NetTest(BitcoinTestFramework): assert_net_servicesnames(int(info[0]["services"], 0x10), info[0]["servicesnames"]) def test_service_flags(self): + self.log.info("Test service flags") self.nodes[0].add_p2p_connection(P2PInterface(), services=(1 << 4) | (1 << 63)) assert_equal(['UNKNOWN[2^4]', 'UNKNOWN[2^63]'], self.nodes[0].getpeerinfo()[-1]['servicesnames']) self.nodes[0].disconnect_p2ps() - def _test_getnodeaddresses(self): + def test_getnodeaddresses(self): + self.log.info("Test getnodeaddresses") self.nodes[0].add_p2p_connection(P2PInterface()) # Add some addresses to the Address Manager over RPC. Due to the way diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 5207b563a1..bd4a53876e 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -32,7 +32,7 @@ from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70016 # past wtxid relay -MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_SUBVERSION = b"/python-p2p-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) MAX_LOCATOR_SZ = 101 diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/p2p.py index eaf637fbb8..57c77e60b5 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/p2p.py @@ -4,10 +4,14 @@ # Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Bitcoin P2P network half-a-node. +"""Test objects for interacting with a bitcoind node over the p2p protocol. -This python code was modified from ArtForz' public domain half-a-node, as -found in the mini-node branch of http://github.com/jgarzik/pynode. +The P2PInterface objects interact with the bitcoind nodes under test using the +node's p2p interface. They can be used to send messages to the node, and +callbacks can be registered that execute when messages are received from the +node. Messages are sent to/received from the node on an asyncio event loop. +State held inside the objects must be guarded by the p2p_lock to avoid data +races between the main testing thread and the event loop. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P @@ -67,7 +71,7 @@ from test_framework.messages import ( ) from test_framework.util import wait_until -logger = logging.getLogger("TestFramework.mininode") +logger = logging.getLogger("TestFramework.p2p") MESSAGEMAP = { b"addr": msg_addr, @@ -320,7 +324,7 @@ class P2PInterface(P2PConnection): We keep a count of how many of each message type has been received and the most recent message of each type.""" - with mininode_lock: + with p2p_lock: try: msgtype = message.msgtype.decode('ascii') self.message_count[msgtype] += 1 @@ -394,7 +398,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return test_function_in() - wait_until(test_function, timeout=timeout, lock=mininode_lock, timeout_factor=self.timeout_factor) + wait_until(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected @@ -498,7 +502,7 @@ class P2PInterface(P2PConnection): # P2PConnection acquires this lock whenever delivering a message to a P2PInterface. # This lock should be acquired in the thread running the test logic to synchronize # access to any data shared with the P2PInterface or P2PConnection. -mininode_lock = threading.Lock() +p2p_lock = threading.Lock() class NetworkThread(threading.Thread): @@ -592,7 +596,7 @@ class P2PDataStore(P2PInterface): - if success is False: assert that the node's tip doesn't advance - if reject_reason is set: assert that the correct reject message is logged""" - with mininode_lock: + with p2p_lock: for block in blocks: self.block_store[block.sha256] = block self.last_block_hash = block.sha256 @@ -629,7 +633,7 @@ class P2PDataStore(P2PInterface): - if expect_disconnect is True: Skip the sync with ping - if reject_reason is set: assert that the correct reject message is logged.""" - with mininode_lock: + with p2p_lock: for tx in txs: self.tx_store[tx.sha256] = tx @@ -668,7 +672,7 @@ class P2PTxInvStore(P2PInterface): self.tx_invs_received[i.hash] += 1 def get_invs(self): - with mininode_lock: + with p2p_lock: return list(self.tx_invs_received.keys()) def wait_for_broadcast(self, txns, timeout=60): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8d402d4888..2a60f8e0c1 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -20,8 +20,8 @@ import time from .authproxy import JSONRPCException from . import coverage +from .p2p import NetworkThread from .test_node import TestNode -from .mininode import NetworkThread from .util import ( MAX_NODES, PortSeed, diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 5eba554a42..5c7a883c43 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -551,7 +551,7 @@ class TestNode(): assert self.p2ps, self._node_msg("No p2p connection") return self.p2ps[0] - def num_connected_mininodes(self): + def num_test_p2p_connections(self): """Return number of test framework p2p connections to the node.""" return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION]) @@ -560,7 +560,7 @@ class TestNode(): for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] - wait_until(lambda: self.num_connected_mininodes() == 0) + wait_until(lambda: self.num_test_p2p_connections() == 0) class TestNodeCLIAttr: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 3362b41209..cfc4ee65d4 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -226,6 +226,14 @@ def satoshi_round(amount): def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): + """Sleep until the predicate resolves to be True. + + Warning: Note that this method is not recommended to be used in tests as it is + not aware of the context of the test framework. Using `wait_until()` counterpart + from `BitcoinTestFramework` or `P2PInterface` class ensures an understandable + amount of timeout and a common shared timeout_factor. Furthermore, `wait_until()` + from `P2PInterface` class in `mininode.py` has a preset lock. + """ if attempts == float('inf') and timeout == float('inf'): timeout = 60 timeout = timeout * timeout_factor diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 53496084ef..56d1da60b7 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -50,6 +50,11 @@ class BumpFeeTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def clear_mempool(self): + # Clear mempool between subtests. The subtests may only depend on chainstate (utxos) + self.nodes[1].generate(1) + self.sync_all() + def run_test(self): # Encrypt wallet for test_locked_wallet_fails test self.nodes[1].encryptwallet(WALLET_PASSPHRASE) @@ -71,7 +76,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() - test_invalid_parameters(rbf_node, dest_address) + self.test_invalid_parameters(rbf_node, dest_address) test_simple_bumpfee_succeeds(self, "default", rbf_node, peer_node, dest_address) test_simple_bumpfee_succeeds(self, "fee_rate", rbf_node, peer_node, dest_address) test_feerate_args(self, rbf_node, peer_node, dest_address) @@ -93,28 +98,30 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) -def test_invalid_parameters(node, dest_address): - txid = spend_one_input(node, dest_address) - # invalid estimate mode - assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { - "estimate_mode": "moo", - }) - assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { - "estimate_mode": 38, - }) - assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { - "estimate_mode": { - "foo": "bar", - }, - }) - assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { - "estimate_mode": Decimal("3.141592"), - }) - # confTarget and conf_target - assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { - "confTarget": 123, - "conf_target": 456, - }) + def test_invalid_parameters(self, node, dest_address): + txid = spend_one_input(node, dest_address) + # invalid estimate mode + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": "moo", + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": 38, + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": { + "foo": "bar", + }, + }) + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": Decimal("3.141592"), + }) + # confTarget and conf_target + assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { + "confTarget": 123, + "conf_target": 456, + }) + self.clear_mempool() + def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.log.info('Test simple bumpfee: {}'.format(mode)) @@ -148,6 +155,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): bumpedwtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + self.clear_mempool() def test_feerate_args(self, rbf_node, peer_node, dest_address): @@ -167,6 +175,7 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) assert_raises_rpc_error(-4, "is too high (cannot be higher than", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) + self.clear_mempool() def test_segwit_bumpfee_succeeds(self, rbf_node, dest_address): @@ -198,12 +207,14 @@ def test_segwit_bumpfee_succeeds(self, rbf_node, dest_address): bumped_tx = rbf_node.bumpfee(rbfid) assert bumped_tx["txid"] in rbf_node.getrawmempool() assert rbfid not in rbf_node.getrawmempool() + self.clear_mempool() def test_nonrbf_bumpfee_fails(self, peer_node, dest_address): self.log.info('Test that we cannot replace a non RBF transaction') not_rbfid = peer_node.sendtoaddress(dest_address, Decimal("0.00090000")) assert_raises_rpc_error(-4, "not BIP 125 replaceable", peer_node.bumpfee, not_rbfid) + self.clear_mempool() def test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address): @@ -211,20 +222,22 @@ def test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address): # here, the rbftx has a peer_node coin and then adds a rbf_node input # Note that this test depends upon the RPC code checking input ownership prior to change outputs # (since it can't use fundrawtransaction, it lacks a proper change output) - utxos = [node.listunspent()[-1] for node in (rbf_node, peer_node)] + fee = Decimal("0.001") + utxos = [node.listunspent(query_options={'minimumAmount': fee})[-1] for node in (rbf_node, peer_node)] inputs = [{ "txid": utxo["txid"], "vout": utxo["vout"], "address": utxo["address"], "sequence": BIP125_SEQUENCE_NUMBER } for utxo in utxos] - output_val = sum(utxo["amount"] for utxo in utxos) - Decimal("0.001") + output_val = sum(utxo["amount"] for utxo in utxos) - fee rawtx = rbf_node.createrawtransaction(inputs, {dest_address: output_val}) signedtx = rbf_node.signrawtransactionwithwallet(rawtx) signedtx = peer_node.signrawtransactionwithwallet(signedtx["hex"]) rbfid = rbf_node.sendrawtransaction(signedtx["hex"]) assert_raises_rpc_error(-4, "Transaction contains inputs that don't belong to this wallet", rbf_node.bumpfee, rbfid) + self.clear_mempool() def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address): @@ -235,6 +248,7 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad tx = rbf_node.signrawtransactionwithwallet(tx) rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + self.clear_mempool() def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): @@ -276,6 +290,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) + self.clear_mempool() def test_dust_to_fee(self, rbf_node, dest_address): @@ -298,6 +313,7 @@ def test_dust_to_fee(self, rbf_node, dest_address): assert_equal(len(fulltx["vout"]), 2) assert_equal(len(full_bumped_tx["vout"]), 1) # change output is eliminated assert_equal(full_bumped_tx["vout"][0]['value'], Decimal("0.00050000")) + self.clear_mempool() def test_settxfee(self, rbf_node, dest_address): @@ -320,6 +336,8 @@ def test_settxfee(self, rbf_node, dest_address): assert_raises_rpc_error(-8, "txfee cannot be more than wallet max tx fee", rbf_node.settxfee, Decimal('0.00003')) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() def test_maxtxfee_fails(self, rbf_node, dest_address): @@ -333,6 +351,8 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): assert_raises_rpc_error(-4, "Unable to create transaction. Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): @@ -415,6 +435,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): rbf_node.unloadwallet("watcher") rbf_node.unloadwallet("signer") + self.clear_mempool() def test_rebumping(self, rbf_node, dest_address): @@ -423,6 +444,7 @@ def test_rebumping(self, rbf_node, dest_address): bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) assert_raises_rpc_error(-4, "already bumped", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL}) rbf_node.bumpfee(bumped["txid"], {"fee_rate": NORMAL}) + self.clear_mempool() def test_rebumping_not_replaceable(self, rbf_node, dest_address): @@ -431,6 +453,7 @@ def test_rebumping_not_replaceable(self, rbf_node, dest_address): bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL, "replaceable": False}) assert_raises_rpc_error(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], {"fee_rate": NORMAL}) + self.clear_mempool() def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): @@ -470,6 +493,7 @@ def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): assert_equal( sum(1 for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid and t["address"] == rbf_node_address and t["spendable"]), 1) + self.clear_mempool() def test_bumpfee_metadata(self, rbf_node, dest_address): @@ -481,6 +505,7 @@ def test_bumpfee_metadata(self, rbf_node, dest_address): bumped_wtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(bumped_wtx["comment"], "comment value") assert_equal(bumped_wtx["to"], "to value") + self.clear_mempool() def test_locked_wallet_fails(self, rbf_node, dest_address): @@ -490,6 +515,7 @@ def test_locked_wallet_fails(self, rbf_node, dest_address): assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first.", rbf_node.bumpfee, rbfid) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.clear_mempool() def test_change_script_match(self, rbf_node, dest_address): @@ -510,6 +536,7 @@ def test_change_script_match(self, rbf_node, dest_address): assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'])) bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) + self.clear_mempool() def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): @@ -548,6 +575,7 @@ def test_no_more_inputs_fails(self, rbf_node, dest_address): # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) + self.clear_mempool() if __name__ == "__main__": diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index b6fe295127..e5c4f12f20 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -15,8 +15,14 @@ from test_framework.util import ( class WalletGroupTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 3 - self.extra_args = [[], [], ['-avoidpartialspends']] + self.num_nodes = 5 + self.extra_args = [ + [], + [], + ["-avoidpartialspends"], + ["-maxapsfee=0.00002719"], + ["-maxapsfee=0.00002720"], + ] self.rpc_timeout = 480 def skip_test_if_missing_module(self): @@ -50,8 +56,8 @@ class WalletGroupTest(BitcoinTestFramework): # one output should be 0.2, the other should be ~0.3 v = [vout["value"] for vout in tx1["vout"]] v.sort() - assert_approx(v[0], 0.2) - assert_approx(v[1], 0.3, 0.0001) + assert_approx(v[0], vexp=0.2, vspan=0.0001) + assert_approx(v[1], vexp=0.3, vspan=0.0001) txid2 = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) tx2 = self.nodes[2].getrawtransaction(txid2, True) @@ -61,8 +67,80 @@ class WalletGroupTest(BitcoinTestFramework): # one output should be 0.2, the other should be ~1.3 v = [vout["value"] for vout in tx2["vout"]] v.sort() - assert_approx(v[0], 0.2) - assert_approx(v[1], 1.3, 0.0001) + assert_approx(v[0], vexp=0.2, vspan=0.0001) + assert_approx(v[1], vexp=1.3, vspan=0.0001) + + # Test 'avoid partial if warranted, even if disabled' + self.sync_all() + self.nodes[0].generate(1) + # Nodes 1-2 now have confirmed UTXOs (letters denote destinations): + # Node #1: Node #2: + # - A 1.0 - D0 1.0 + # - B0 1.0 - D1 0.5 + # - B1 0.5 - E0 1.0 + # - C0 1.0 - E1 0.5 + # - C1 0.5 - F ~1.3 + # - D ~0.3 + assert_approx(self.nodes[1].getbalance(), vexp=4.3, vspan=0.0001) + assert_approx(self.nodes[2].getbalance(), vexp=4.3, vspan=0.0001) + # Sending 1.4 btc should pick one 1.0 + one more. For node #1, + # this could be (A / B0 / C0) + (B1 / C1 / D). We ensure that it is + # B0 + B1 or C0 + C1, because this avoids partial spends while not being + # detrimental to transaction cost + txid3 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.4) + tx3 = self.nodes[1].getrawtransaction(txid3, True) + # tx3 should have 2 inputs and 2 outputs + assert_equal(2, len(tx3["vin"])) + assert_equal(2, len(tx3["vout"])) + # the accumulated value should be 1.5, so the outputs should be + # ~0.1 and 1.4 and should come from the same destination + values = [vout["value"] for vout in tx3["vout"]] + values.sort() + assert_approx(values[0], vexp=0.1, vspan=0.0001) + assert_approx(values[1], vexp=1.4, vspan=0.0001) + + input_txids = [vin["txid"] for vin in tx3["vin"]] + input_addrs = [self.nodes[1].gettransaction(txid)['details'][0]['address'] for txid in input_txids] + assert_equal(input_addrs[0], input_addrs[1]) + # Node 2 enforces avoidpartialspends so needs no checking here + + # Test wallet option maxapsfee with Node 3 + addr_aps = self.nodes[3].getnewaddress() + self.nodes[0].sendtoaddress(addr_aps, 1.0) + self.nodes[0].sendtoaddress(addr_aps, 1.0) + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[3].assert_debug_log(['Fee non-grouped = 2820, grouped = 4160, using grouped']): + txid4 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 0.1) + tx4 = self.nodes[3].getrawtransaction(txid4, True) + # tx4 should have 2 inputs and 2 outputs although one output would + # have been enough and the transaction caused higher fees + assert_equal(2, len(tx4["vin"])) + assert_equal(2, len(tx4["vout"])) + + addr_aps2 = self.nodes[3].getnewaddress() + [self.nodes[0].sendtoaddress(addr_aps2, 1.0) for _ in range(5)] + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[3].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using non-grouped']): + txid5 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) + tx5 = self.nodes[3].getrawtransaction(txid5, True) + # tx5 should have 3 inputs (1.0, 1.0, 1.0) and 2 outputs + assert_equal(3, len(tx5["vin"])) + assert_equal(2, len(tx5["vout"])) + + # Test wallet option maxapsfee with node 4, which sets maxapsfee + # 1 sat higher, crossing the threshold from non-grouped to grouped. + addr_aps3 = self.nodes[4].getnewaddress() + [self.nodes[0].sendtoaddress(addr_aps3, 1.0) for _ in range(5)] + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[4].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using grouped']): + txid6 = self.nodes[4].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) + tx6 = self.nodes[4].getrawtransaction(txid6, True) + # tx6 should have 5 inputs and 2 outputs + assert_equal(5, len(tx6["vin"])) + assert_equal(2, len(tx6["vout"])) # Empty out node2's wallet self.nodes[2].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=self.nodes[2].getbalance(), subtractfeefromamount=True) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 3417616d77..0327c9e070 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -7,9 +7,9 @@ import time from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ToHex -from test_framework.mininode import P2PTxInvStore, mininode_lock +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, wait_until +from test_framework.util import assert_equal class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): @@ -24,7 +24,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a new transaction and wait until it's broadcast") - txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + txid = node.sendtoaddress(node.getnewaddress(), 1) # Wallet rebroadcast is first scheduled 1 sec after startup (see # nNextResend in ResendWalletTransactions()). Sleep for just over a @@ -33,7 +33,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): time.sleep(1.1) # Can take a few seconds due to transaction trickling - wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + node.p2p.wait_for_broadcast([txid]) # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) node.add_p2p_connection(P2PTxInvStore()) @@ -58,13 +58,13 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): two_min = 2 * 60 node.setmocktime(now + twelve_hrs - two_min) time.sleep(2) # ensure enough time has passed for rebroadcast attempt to occur - assert_equal(txid in node.p2ps[1].get_invs(), False) + assert_equal(int(txid, 16) in node.p2ps[1].get_invs(), False) self.log.info("Bump time & check that transaction is rebroadcast") # Transaction should be rebroadcast approximately 24 hours in the future, # but can range from 12-36. So bump 36 hours to be sure. node.setmocktime(now + 36 * 60 * 60) - wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) + node.p2p.wait_for_broadcast([txid]) if __name__ == '__main__': diff --git a/test/functional/wallet_zapwallettxes.py b/test/functional/wallet_zapwallettxes.py index 7f1cdbd20b..1287092cac 100755 --- a/test/functional/wallet_zapwallettxes.py +++ b/test/functional/wallet_zapwallettxes.py @@ -18,9 +18,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - wait_until, ) + class ZapWalletTXesTest (BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -59,7 +59,7 @@ class ZapWalletTXesTest (BitcoinTestFramework): # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. self.restart_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) - wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) + self.wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) self.nodes[0].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 56b18752ec..c7895edbcc 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -56,6 +56,14 @@ def main(): '--m_dir', help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', ) + parser.add_argument( + '-g', + '--generate', + action='store_true', + help='Create new corpus seeds (or extend the existing ones) by running' + ' the given targets for a finite number of times. Outputs them to' + ' the passed seed_dir.' + ) args = parser.parse_args() @@ -100,19 +108,20 @@ def main(): logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection))) - test_list_seedless = [] - for t in test_list_selection: - corpus_path = os.path.join(args.seed_dir, t) - if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: - test_list_seedless.append(t) - test_list_seedless.sort() - if test_list_seedless: - logging.info( - "Fuzzing harnesses lacking a seed corpus: {}".format( - " ".join(test_list_seedless) + if not args.generate: + test_list_seedless = [] + for t in test_list_selection: + corpus_path = os.path.join(args.seed_dir, t) + if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: + test_list_seedless.append(t) + test_list_seedless.sort() + if test_list_seedless: + logging.info( + "Fuzzing harnesses lacking a seed corpus: {}".format( + " ".join(test_list_seedless) + ) ) - ) - logging.info("Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets") + logging.info("Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets") try: help_output = subprocess.run( @@ -133,6 +142,14 @@ def main(): sys.exit(1) with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: + if args.generate: + return generate_corpus_seeds( + fuzz_pool=fuzz_pool, + build_dir=config["environment"]["BUILDDIR"], + seed_dir=args.seed_dir, + targets=test_list_selection, + ) + if args.m_dir: merge_inputs( fuzz_pool=fuzz_pool, @@ -152,6 +169,37 @@ def main(): ) +def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): + """Generates new corpus seeds. + + Run {targets} without input, and outputs the generated corpus seeds to + {seed_dir}. + """ + logging.info("Generating corpus seeds to {}".format(seed_dir)) + + def job(command): + logging.debug("Running '{}'\n".format(" ".join(command))) + logging.debug("Command '{}' output:\n'{}'\n".format( + ' '.join(command), + subprocess.run(command, check=True, stderr=subprocess.PIPE, + universal_newlines=True).stderr + )) + + futures = [] + for target in targets: + target_seed_dir = os.path.join(seed_dir, target) + os.makedirs(target_seed_dir, exist_ok=True) + command = [ + os.path.join(build_dir, "src", "test", "fuzz", target), + "-runs=100000", + target_seed_dir, + ] + futures.append(fuzz_pool.submit(job, command)) + + for future in as_completed(futures): + future.result() + + def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] |